diff --git a/build.rs b/build.rs index cb426a9d0..705b48546 100644 --- a/build.rs +++ b/build.rs @@ -276,6 +276,10 @@ mod asm { let mut nasm = nasm_rs::Build::new(); nasm.min_version(2, 14, 0); nasm.files(asm_file_paths); + #[cfg(debug_assertions)] + nasm.flag("-g"); + #[cfg(debug_assertions)] + nasm.flag("-Fdwarf"); nasm.flag(&format!("-I{}/", out_dir.to_str().unwrap())); nasm.flag("-Isrc/"); let obj = nasm.compile_objects().unwrap_or_else(|e| { @@ -295,6 +299,7 @@ mod asm { .files(asm_file_paths) .include(".") .include(&out_dir) + .debug(cfg!(debug_assertions)) .compile(rav1dasm); } diff --git a/include/common/bitdepth.rs b/include/common/bitdepth.rs index 06568ea73..888e52de1 100644 --- a/include/common/bitdepth.rs +++ b/include/common/bitdepth.rs @@ -375,8 +375,8 @@ impl BitDepth for BitDepth16 { self.bitdepth_max } - /// 4 for 10 bits/component. - /// 2 for 12 bits/component. + /// - 4 for 10 bits/component. + /// - 2 for 12 bits/component. fn get_intermediate_bits(&self) -> u8 { 14 - self.bitdepth() } @@ -568,6 +568,26 @@ macro_rules! bd_fn { }}; } +/// Select and declare a [`BitDepth`]-dependent `extern "C" fn`. +/// +/// Similar to [`bd_fn!`] except that it selects which [`BitDepth`] `fn` +/// based on `$bpc:literal bpc` instead of `$BD:ty`. +macro_rules! bpc_fn { + ($bpc:literal bpc, $name:ident, $asm:ident) => {{ + use $crate::include::common::bitdepth::fn_identity; + + bpc_fn!(fn_identity, $bpc bpc, $name, $asm) + }}; + + ($decl_fn:path, $bpc:literal bpc, $name:ident, $asm:ident) => {{ + use paste::paste; + + paste! { + $decl_fn!(fn []) + } + }}; +} + #[cfg(feature = "asm")] macro_rules! fn_identity { (fn $name:ident) => { @@ -578,5 +598,8 @@ macro_rules! fn_identity { #[cfg(feature = "asm")] pub(crate) use bd_fn; +#[cfg(feature = "asm")] +pub(crate) use bpc_fn; + #[cfg(feature = "asm")] pub(crate) use fn_identity; diff --git a/lib.rs b/lib.rs index aced8a7a4..5caf77019 100644 --- a/lib.rs +++ b/lib.rs @@ -54,9 +54,6 @@ pub mod src { mod ipred_prepare; mod itx; mod itx_1d; - #[cfg(feature = "bitdepth_16")] - mod itx_tmpl_16; - mod itx_tmpl_8; mod levels; mod lf_apply; mod lf_mask; diff --git a/src/decode.rs b/src/decode.rs index 815a8edb6..d042cb75b 100644 --- a/src/decode.rs +++ b/src/decode.rs @@ -74,6 +74,7 @@ use crate::src::intra_edge::EdgeFlags; use crate::src::intra_edge::EdgeIndex; use crate::src::intra_edge::IntraEdges; use crate::src::ipred::rav1d_intra_pred_dsp_init; +use crate::src::itx::rav1d_itx_dsp_init; use crate::src::levels::mv; use crate::src::levels::Av1Block; use crate::src::levels::BS_128x128; @@ -217,12 +218,6 @@ use std::slice; use std::sync::atomic::AtomicI32; use std::sync::atomic::Ordering; -#[cfg(feature = "bitdepth_8")] -use crate::src::itx_tmpl_8::rav1d_itx_dsp_init_8bpc; - -#[cfg(feature = "bitdepth_16")] -use crate::src::itx_tmpl_16::rav1d_itx_dsp_init_16bpc; - fn init_quant_tables( seq_hdr: &Rav1dSequenceHeader, frame_hdr: &Rav1dFrameHeader, @@ -4959,7 +4954,7 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult { 8 => { rav1d_cdef_dsp_init::(&mut dsp.cdef); rav1d_intra_pred_dsp_init::(&mut dsp.ipred); - rav1d_itx_dsp_init_8bpc(&mut dsp.itx, bpc); + rav1d_itx_dsp_init::(&mut dsp.itx, bpc); rav1d_loop_filter_dsp_init::(&mut dsp.lf); rav1d_loop_restoration_dsp_init::(&mut dsp.lr, bpc); rav1d_mc_dsp_init::(&mut dsp.mc); @@ -4969,7 +4964,7 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult { 10 | 12 => { rav1d_cdef_dsp_init::(&mut dsp.cdef); rav1d_intra_pred_dsp_init::(&mut dsp.ipred); - rav1d_itx_dsp_init_16bpc(&mut dsp.itx, bpc); + rav1d_itx_dsp_init::(&mut dsp.itx, bpc); rav1d_loop_filter_dsp_init::(&mut dsp.lf); rav1d_loop_restoration_dsp_init::(&mut dsp.lr, bpc); rav1d_mc_dsp_init::(&mut dsp.mc); diff --git a/src/itx.rs b/src/itx.rs index 3f8f6435a..d20f04291 100644 --- a/src/itx.rs +++ b/src/itx.rs @@ -3,14 +3,61 @@ use crate::include::common::bitdepth::BitDepth; use crate::include::common::bitdepth::DynCoef; use crate::include::common::bitdepth::DynPixel; use crate::include::common::intops::iclip; +use crate::src::levels::ADST_ADST; +use crate::src::levels::ADST_DCT; +use crate::src::levels::ADST_FLIPADST; +use crate::src::levels::DCT_ADST; +use crate::src::levels::DCT_DCT; +use crate::src::levels::DCT_FLIPADST; +use crate::src::levels::FLIPADST_ADST; +use crate::src::levels::FLIPADST_DCT; +use crate::src::levels::FLIPADST_FLIPADST; +use crate::src::levels::H_ADST; +use crate::src::levels::H_DCT; +use crate::src::levels::H_FLIPADST; +use crate::src::levels::IDTX; use crate::src::levels::N_RECT_TX_SIZES; use crate::src::levels::N_TX_TYPES_PLUS_LL; +use crate::src::levels::RTX_16X32; +use crate::src::levels::RTX_16X4; +use crate::src::levels::RTX_16X64; +use crate::src::levels::RTX_16X8; +use crate::src::levels::RTX_32X16; +use crate::src::levels::RTX_32X64; +use crate::src::levels::RTX_32X8; +use crate::src::levels::RTX_4X16; +use crate::src::levels::RTX_4X8; +use crate::src::levels::RTX_64X16; +use crate::src::levels::RTX_64X32; +use crate::src::levels::RTX_8X16; +use crate::src::levels::RTX_8X32; +use crate::src::levels::RTX_8X4; +use crate::src::levels::TX_16X16; +use crate::src::levels::TX_32X32; +use crate::src::levels::TX_4X4; +use crate::src::levels::TX_64X64; +use crate::src::levels::TX_8X8; +use crate::src::levels::V_ADST; +use crate::src::levels::V_DCT; +use crate::src::levels::V_FLIPADST; +use crate::src::levels::WHT_WHT; use libc::memset; use libc::ptrdiff_t; use std::cmp; use std::ffi::c_int; use std::ffi::c_void; +#[cfg(feature = "asm")] +use crate::src::cpu::{rav1d_get_cpu_flags, CpuFlags}; + +#[cfg(feature = "asm")] +use cfg_if::cfg_if; + +#[cfg(feature = "asm")] +use crate::include::common::bitdepth::bd_fn; +#[cfg(feature = "asm")] +use crate::include::common::bitdepth::bpc_fn; + pub type itx_1d_fn = Option ()>; pub unsafe fn inv_txfm_add_rust( @@ -150,8 +197,7 @@ pub struct Rav1dInvTxfmDSPContext { #[cfg(feature = "asm")] macro_rules! decl_itx_fn { ($name:ident) => { - // TODO(legare): Temporarily pub until init fns are deduplicated. - pub(crate) fn $name( + fn $name( dst: *mut DynPixel, dst_stride: ptrdiff_t, coeff: *mut DynCoef, @@ -324,8 +370,7 @@ extern "C" { macro_rules! inv_txfm_fn { ($type1:ident, $type2:ident, $w:literal, $h:literal, $shift:literal, $has_dconly:literal) => { paste::paste! { - // TODO(legare): Temporarily pub until init fns are deduplicated. - pub(crate) unsafe extern "C" fn [] ( + unsafe extern "C" fn [] ( dst: *mut DynPixel, stride: ptrdiff_t, coeff: *mut DynCoef, @@ -409,3 +454,651 @@ inv_txfm_fn64!(32, 64, 1); inv_txfm_fn64!(64, 16, 2); inv_txfm_fn64!(64, 32, 1); inv_txfm_fn64!(64, 64, 2); + +pub(crate) unsafe extern "C" fn inv_txfm_add_wht_wht_4x4_c_erased( + dst: *mut DynPixel, + stride: ptrdiff_t, + coeff: *mut DynCoef, + eob: c_int, + bitdepth_max: c_int, +) { + inv_txfm_add_wht_wht_4x4_rust::( + dst.cast(), + stride, + coeff.cast(), + eob, + BD::from_c(bitdepth_max), + ); +} + +unsafe fn inv_txfm_add_wht_wht_4x4_rust( + mut dst: *mut BD::Pixel, + stride: ptrdiff_t, + coeff: *mut BD::Coef, + _eob: c_int, + bd: BD, +) { + use crate::src::itx_1d::dav1d_inv_wht4_1d_c; + + let mut tmp: [i32; 16] = [0; 16]; + let mut c: *mut i32 = tmp.as_mut_ptr(); + let mut y = 0; + while y < 4 { + let mut x = 0; + while x < 4 { + *c.offset(x as isize) = (*coeff.offset((y + x * 4) as isize)).as_::() >> 2; + x += 1; + } + dav1d_inv_wht4_1d_c(c, 1 as c_int as ptrdiff_t); + y += 1; + c = c.offset(4); + } + memset( + coeff as *mut c_void, + 0 as c_int, + ::core::mem::size_of::() + .wrapping_mul(4) + .wrapping_mul(4), + ); + let mut x_0 = 0; + while x_0 < 4 { + dav1d_inv_wht4_1d_c( + &mut *tmp.as_mut_ptr().offset(x_0 as isize), + 4 as c_int as ptrdiff_t, + ); + x_0 += 1; + } + c = tmp.as_mut_ptr(); + let mut y_0 = 0; + while y_0 < 4 { + let mut x_1 = 0; + while x_1 < 4 { + let fresh1 = c; + c = c.offset(1); + *dst.offset(x_1 as isize) = + bd.iclip_pixel((*dst.offset(x_1 as isize)).as_::() + *fresh1); + x_1 += 1; + } + y_0 += 1; + dst = dst.offset(BD::pxstride(stride as usize) as isize); + } +} + +#[cfg(feature = "asm")] +macro_rules! assign_itx_fn { + ($c:ident, $BD:ty, $w:literal, $h:literal, $type:ident, $type_enum:ident, $ext:ident) => {{ + use paste::paste; + + paste! { + (*$c).itxfm_add[[] as usize][$type_enum as usize] + = Some(bd_fn!(BD, [< inv_txfm_add_ $type _ $w x $h >], $ext)); + } + }}; + + ($c:ident, $BD:ty, $pfx:ident, $w:literal, $h:literal, $type:ident, $type_enum:ident, $ext:ident) => {{ + use paste::paste; + + paste! { + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][$type_enum as usize] + = Some(bd_fn!(BD, [< inv_txfm_add_ $type _ $w x $h >], $ext)); + } + }}; +} + +#[cfg(all(feature = "asm", any(target_arch = "x86", target_arch = "x86_64")))] +macro_rules! assign_itx_bpc_fn { + ($c:ident, $pfx:ident, $w:literal, $h:literal, $type:ident, $type_enum:ident, $bpc:literal bpc, $ext:ident) => {{ + use paste::paste; + + paste! { + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][$type_enum as usize] + = Some(bpc_fn!($bpc bpc, [< inv_txfm_add_ $type _ $w x $h >], $ext)); + } + }}; + + ($c:ident, $w:literal, $h:literal, $type:ident, $type_enum:ident, $bpc:literal bpc, $ext:ident) => {{ + use paste::paste; + + paste! { + (*$c).itxfm_add[[] as usize][$type_enum as usize] + = Some(bpc_fn!($bpc bpc, [< inv_txfm_add_ $type _ $w x $h >], $ext)); + } + }}; +} + +#[cfg(all(feature = "asm", any(target_arch = "x86", target_arch = "x86_64")))] +macro_rules! assign_itx1_bpc_fn { + ($c:ident, $w:literal, $h:literal, $bpc:literal bpc, $ext:ident) => {{ + assign_itx_bpc_fn!($c, $w, $h, dct_dct, DCT_DCT, $bpc bpc, $ext) + }}; + + ($c:ident, $pfx:ident, $w:literal, $h:literal, $bpc:literal bpc, $ext:ident) => {{ + assign_itx_bpc_fn!($c, $pfx, $w, $h, dct_dct, DCT_DCT, $bpc bpc, $ext) + }}; +} + +#[cfg(all(feature = "asm", any(target_arch = "arm", target_arch = "aarch64")))] +macro_rules! assign_itx1_fn { + ($c:ident, $BD:ty, $w:literal, $h:literal, $ext:ident) => {{ + assign_itx_fn!($c, BD, $w, $h, dct_dct, DCT_DCT, $ext) + }}; + + ($c:ident, $BD:ty, $pfx:ident, $w:literal, $h:literal, $ext:ident) => {{ + assign_itx_fn!($c, BD, $pfx, $w, $h, dct_dct, DCT_DCT, $ext) + }}; +} + +#[cfg(all(feature = "asm", any(target_arch = "x86", target_arch = "x86_64")))] +macro_rules! assign_itx2_bpc_fn { + ($c:ident, $w:literal, $h:literal, $bpc:literal bpc, $ext:ident) => {{ + assign_itx1_bpc_fn!($c, $w, $h, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, identity_identity, IDTX, $bpc bpc, $ext) + }}; + + ($c:ident, $pfx:ident, $w:literal, $h:literal, $bpc:literal bpc, $ext:ident) => {{ + assign_itx1_bpc_fn!($c, $pfx, $w, $h, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, identity_identity, IDTX, $bpc bpc, $ext) + }}; +} + +#[cfg(all(feature = "asm", any(target_arch = "arm", target_arch = "aarch64")))] +macro_rules! assign_itx2_fn { + ($c:ident, $BD:ty, $w:literal, $h:literal, $ext:ident) => {{ + assign_itx1_fn!($c, BD, $w, $h, $ext); + assign_itx_fn!($c, BD, $w, $h, identity_identity, IDTX, $ext) + }}; + + ($c:ident, $BD:ty, $pfx:ident, $w:literal, $h:literal, $ext:ident) => {{ + assign_itx1_fn!($c, BD, $pfx, $w, $h, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, identity_identity, IDTX, $ext) + }}; +} + +#[cfg(all(feature = "asm", any(target_arch = "x86", target_arch = "x86_64")))] +#[rustfmt::skip] +macro_rules! assign_itx12_bpc_fn { + ($c:ident, $w:literal, $h:literal, $bpc:literal bpc, $ext:ident) => {{ + assign_itx2_bpc_fn!($c, $w, $h, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, dct_adst, ADST_DCT, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, dct_flipadst, FLIPADST_DCT, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, dct_identity, H_DCT, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, adst_dct, DCT_ADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, adst_adst, ADST_ADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, adst_flipadst, FLIPADST_ADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, flipadst_dct, DCT_FLIPADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, flipadst_adst, ADST_FLIPADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, flipadst_flipadst, FLIPADST_FLIPADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, identity_dct, V_DCT, $bpc bpc, $ext); + + }}; + + ($c:ident, $pfx:ident, $w:literal, $h:literal, $bpc:literal bpc, $ext:ident) => {{ + assign_itx2_bpc_fn!($c, $pfx, $w, $h, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, dct_adst, ADST_DCT, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, dct_flipadst, FLIPADST_DCT, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, dct_identity, H_DCT, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, adst_dct, DCT_ADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, adst_adst, ADST_ADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, adst_flipadst, FLIPADST_ADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, flipadst_dct, DCT_FLIPADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, flipadst_adst, ADST_FLIPADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, flipadst_flipadst, FLIPADST_FLIPADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, identity_dct, V_DCT, $bpc bpc, $ext); + }}; +} + +#[cfg(all(feature = "asm", any(target_arch = "arm", target_arch = "aarch64")))] +#[rustfmt::skip] +macro_rules! assign_itx12_fn { + ($c:ident, $BD:ty, $w:literal, $h:literal, $ext:ident) => {{ + assign_itx2_fn!($c, BD, $w, $h, $ext); + assign_itx_fn!($c, BD, $w, $h, dct_flipadst, FLIPADST_DCT, $ext); + assign_itx_fn!($c, BD, $w, $h, dct_adst, ADST_DCT, $ext); + assign_itx_fn!($c, BD, $w, $h, dct_identity, H_DCT, $ext); + assign_itx_fn!($c, BD, $w, $h, adst_dct, DCT_ADST, $ext); + assign_itx_fn!($c, BD, $w, $h, adst_adst, ADST_ADST, $ext); + assign_itx_fn!($c, BD, $w, $h, adst_flipadst, FLIPADST_ADST, $ext); + assign_itx_fn!($c, BD, $w, $h, flipadst_dct, DCT_FLIPADST, $ext); + assign_itx_fn!($c, BD, $w, $h, flipadst_adst, ADST_FLIPADST, $ext); + assign_itx_fn!($c, BD, $w, $h, flipadst_flipadst, FLIPADST_FLIPADST, $ext); + assign_itx_fn!($c, BD, $w, $h, identity_dct, V_DCT, $ext); + }}; + + ($c:ident, $BD:ty, $pfx:ident, $w:literal, $h:literal, $ext:ident) => {{ + assign_itx2_fn!($c, BD, $pfx, $w, $h, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, dct_flipadst, FLIPADST_DCT, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, dct_adst, ADST_DCT, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, dct_identity, H_DCT, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, adst_dct, DCT_ADST, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, adst_adst, ADST_ADST, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, adst_flipadst, FLIPADST_ADST, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, flipadst_dct, DCT_FLIPADST, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, flipadst_adst, ADST_FLIPADST, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, flipadst_flipadst, FLIPADST_FLIPADST, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, identity_dct, V_DCT, $ext); + }}; +} + +#[cfg(all(feature = "asm", any(target_arch = "x86", target_arch = "x86_64")))] +macro_rules! assign_itx16_bpc_fn { + ($c:ident, $w:literal, $h:literal, $bpc:literal bpc, $ext:ident) => {{ + assign_itx12_bpc_fn!($c, $w, $h, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, adst_identity, H_ADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, flipadst_identity, H_FLIPADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, identity_adst, V_ADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $w, $h, identity_flipadst, V_FLIPADST, $bpc bpc, $ext); + }}; + + ($c:ident, $pfx:ident, $w:literal, $h:literal, $bpc:literal bpc, $ext:ident) => {{ + assign_itx12_bpc_fn!($c, $pfx, $w, $h, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, adst_identity, H_ADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, flipadst_identity, H_FLIPADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, identity_adst, V_ADST, $bpc bpc, $ext); + assign_itx_bpc_fn!($c, $pfx, $w, $h, identity_flipadst, V_FLIPADST, $bpc bpc, $ext); + }}; +} + +#[cfg(all(feature = "asm", any(target_arch = "arm", target_arch = "aarch64")))] +macro_rules! assign_itx16_fn { + ($c:ident, $BD:ty, $w:literal, $h:literal, $ext:ident) => {{ + assign_itx12_fn!($c, BD, $w, $h, $ext); + assign_itx_fn!($c, BD, $w, $h, adst_identity, H_ADST, $ext); + assign_itx_fn!($c, BD, $w, $h, flipadst_identity, H_FLIPADST, $ext); + assign_itx_fn!($c, BD, $w, $h, identity_adst, V_ADST, $ext); + assign_itx_fn!($c, BD, $w, $h, identity_flipadst, V_FLIPADST, $ext); + }}; + + ($c:ident, $BD:ty, $pfx:ident, $w:literal, $h:literal, $ext:ident) => {{ + assign_itx12_fn!($c, BD, $pfx, $w, $h, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, adst_identity, H_ADST, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, flipadst_identity, H_FLIPADST, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, identity_adst, V_ADST, $ext); + assign_itx_fn!($c, BD, $pfx, $w, $h, identity_flipadst, V_FLIPADST, $ext); + }}; +} + +#[cfg(all(feature = "asm", any(target_arch = "x86", target_arch = "x86_64")))] +#[inline(always)] +#[rustfmt::skip] +unsafe fn itx_dsp_init_x86(c: *mut Rav1dInvTxfmDSPContext, bpc: c_int) { + + let flags = rav1d_get_cpu_flags(); + + if !flags.contains(CpuFlags::SSE2) { + return; + } + + assign_itx_fn!(c, BD, 4, 4, wht_wht, WHT_WHT, sse2); + + if !flags.contains(CpuFlags::SSSE3) { + return; + } + + if BD::BITDEPTH == 8 { + assign_itx16_bpc_fn!(c, 4, 4, 8 bpc, ssse3); + assign_itx16_bpc_fn!(c, R, 4, 8, 8 bpc, ssse3); + assign_itx16_bpc_fn!(c, R, 8, 4, 8 bpc, ssse3); + assign_itx16_bpc_fn!(c, 8, 8, 8 bpc, ssse3); + assign_itx16_bpc_fn!(c, R, 4, 16, 8 bpc, ssse3); + assign_itx16_bpc_fn!(c, R, 16, 4, 8 bpc, ssse3); + assign_itx16_bpc_fn!(c, R, 8, 16, 8 bpc, ssse3); + assign_itx16_bpc_fn!(c, R, 16, 8, 8 bpc, ssse3); + assign_itx12_bpc_fn!(c, 16, 16, 8 bpc, ssse3); + assign_itx2_bpc_fn! (c, R, 8, 32, 8 bpc, ssse3); + assign_itx2_bpc_fn! (c, R, 32, 8, 8 bpc, ssse3); + assign_itx2_bpc_fn! (c, R, 16, 32, 8 bpc, ssse3); + assign_itx2_bpc_fn! (c, R, 32, 16, 8 bpc, ssse3); + assign_itx2_bpc_fn! (c, 32, 32, 8 bpc, ssse3); + assign_itx1_bpc_fn! (c, R, 16, 64, 8 bpc, ssse3); + assign_itx1_bpc_fn! (c, R, 32, 64, 8 bpc, ssse3); + assign_itx1_bpc_fn! (c, R, 64, 16, 8 bpc, ssse3); + assign_itx1_bpc_fn! (c, R, 64, 32, 8 bpc, ssse3); + assign_itx1_bpc_fn! (c, 64, 64, 8 bpc, ssse3); + } + + if !flags.contains(CpuFlags::SSE41) { + return; + } + + if BD::BITDEPTH == 16 { + if bpc == 10 { + assign_itx16_bpc_fn!(c, 4, 4, 16 bpc, sse4); + assign_itx16_bpc_fn!(c, R, 4, 8, 16 bpc, sse4); + assign_itx16_bpc_fn!(c, R, 4, 16, 16 bpc, sse4); + assign_itx16_bpc_fn!(c, R, 8, 4, 16 bpc, sse4); + assign_itx16_bpc_fn!(c, 8, 8, 16 bpc, sse4); + assign_itx16_bpc_fn!(c, R, 8, 16, 16 bpc, sse4); + assign_itx16_bpc_fn!(c, R, 16, 4, 16 bpc, sse4); + assign_itx16_bpc_fn!(c, R, 16, 8, 16 bpc, sse4); + assign_itx12_bpc_fn!(c, 16, 16, 16 bpc, sse4); + assign_itx2_bpc_fn! (c, R, 8, 32, 16 bpc, sse4); + assign_itx2_bpc_fn! (c, R, 16, 32, 16 bpc, sse4); + assign_itx2_bpc_fn! (c, R, 32, 8, 16 bpc, sse4); + assign_itx2_bpc_fn! (c, R, 32, 16, 16 bpc, sse4); + assign_itx2_bpc_fn! (c, 32, 32, 16 bpc, sse4); + assign_itx1_bpc_fn! (c, R, 16, 64, 16 bpc, sse4); + assign_itx1_bpc_fn! (c, R, 32, 64, 16 bpc, sse4); + assign_itx1_bpc_fn! (c, R, 64, 16, 16 bpc, sse4); + assign_itx1_bpc_fn! (c, R, 64, 32, 16 bpc, sse4); + assign_itx1_bpc_fn! (c, 64, 64, 16 bpc, sse4); + } + } + + #[cfg(target_arch = "x86_64")] + { + if !flags.contains(CpuFlags::AVX2) { + return; + } + + assign_itx_fn!(c, BD, 4, 4, wht_wht, WHT_WHT, avx2); + + if BD::BITDEPTH == 8 { + assign_itx16_bpc_fn!(c, 4, 4, 8 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 4, 8, 8 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 4, 16, 8 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 8, 4, 8 bpc, avx2); + assign_itx16_bpc_fn!(c, 8, 8, 8 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 8, 16, 8 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 16, 4, 8 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 16, 8, 8 bpc, avx2); + assign_itx12_bpc_fn!(c, 16, 16, 8 bpc, avx2); + assign_itx2_bpc_fn! (c, R, 8, 32, 8 bpc, avx2); + assign_itx2_bpc_fn! (c, R, 16, 32, 8 bpc, avx2); + assign_itx2_bpc_fn! (c, R, 32, 8, 8 bpc, avx2); + assign_itx2_bpc_fn! (c, R, 32, 16, 8 bpc, avx2); + assign_itx2_bpc_fn! (c, 32, 32, 8 bpc, avx2); + assign_itx1_bpc_fn! (c, R, 16, 64, 8 bpc, avx2); + assign_itx1_bpc_fn! (c, R, 32, 64, 8 bpc, avx2); + assign_itx1_bpc_fn! (c, R, 64, 16, 8 bpc, avx2); + assign_itx1_bpc_fn! (c, R, 64, 32, 8 bpc, avx2); + assign_itx1_bpc_fn! (c, 64, 64, 8 bpc, avx2); + } else { + if bpc == 10 { + assign_itx16_bpc_fn!(c, 4, 4, 10 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 4, 8, 10 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 4, 16, 10 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 8, 4, 10 bpc, avx2); + assign_itx16_bpc_fn!(c, 8, 8, 10 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 8, 16, 10 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 16, 4, 10 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 16, 8, 10 bpc, avx2); + assign_itx12_bpc_fn!(c, 16, 16, 10 bpc, avx2); + assign_itx2_bpc_fn! (c, R, 8, 32, 10 bpc, avx2); + assign_itx2_bpc_fn! (c, R, 16, 32, 10 bpc, avx2); + assign_itx2_bpc_fn! (c, R, 32, 8, 10 bpc, avx2); + assign_itx2_bpc_fn! (c, R, 32, 16, 10 bpc, avx2); + assign_itx2_bpc_fn! (c, 32, 32, 10 bpc, avx2); + assign_itx1_bpc_fn! (c, R, 16, 64, 10 bpc, avx2); + assign_itx1_bpc_fn! (c, R, 32, 64, 10 bpc, avx2); + assign_itx1_bpc_fn! (c, R, 64, 16, 10 bpc, avx2); + assign_itx1_bpc_fn! (c, R, 64, 32, 10 bpc, avx2); + assign_itx1_bpc_fn! (c, 64, 64, 10 bpc, avx2); + } else { + assign_itx16_bpc_fn!(c, 4, 4, 12 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 4, 8, 12 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 4, 16, 12 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 8, 4, 12 bpc, avx2); + assign_itx16_bpc_fn!(c, 8, 8, 12 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 8, 16, 12 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 16, 4, 12 bpc, avx2); + assign_itx16_bpc_fn!(c, R, 16, 8, 12 bpc, avx2); + assign_itx12_bpc_fn!(c, 16, 16, 12 bpc, avx2); + assign_itx2_bpc_fn! (c, R, 8, 32, 12 bpc, avx2); + assign_itx2_bpc_fn! (c, R, 32, 8, 12 bpc, avx2); + assign_itx_bpc_fn! (c, R, 16, 32, identity_identity, IDTX, 12 bpc, avx2); + assign_itx_bpc_fn! (c, R, 32, 16, identity_identity, IDTX, 12 bpc, avx2); + assign_itx_bpc_fn! (c, 32, 32, identity_identity, IDTX, 12 bpc, avx2); + } + } + + if !flags.contains(CpuFlags::AVX512ICL) { + return; + } + + if BD::BITDEPTH == 8 { + assign_itx16_bpc_fn!(c, 4, 4, 8 bpc, avx512icl); // no wht + assign_itx16_bpc_fn!(c, R, 4, 8, 8 bpc, avx512icl); + assign_itx16_bpc_fn!(c, R, 4, 16, 8 bpc, avx512icl); + assign_itx16_bpc_fn!(c, R, 8, 4, 8 bpc, avx512icl); + assign_itx16_bpc_fn!(c, 8, 8, 8 bpc, avx512icl); + assign_itx16_bpc_fn!(c, R, 8, 16, 8 bpc, avx512icl); + assign_itx16_bpc_fn!(c, R, 16, 4, 8 bpc, avx512icl); + assign_itx16_bpc_fn!(c, R, 16, 8, 8 bpc, avx512icl); + assign_itx12_bpc_fn!(c, 16, 16, 8 bpc, avx512icl); + assign_itx2_bpc_fn! (c, R, 8, 32, 8 bpc, avx512icl); + assign_itx2_bpc_fn! (c, R, 16, 32, 8 bpc, avx512icl); + assign_itx2_bpc_fn! (c, R, 32, 8, 8 bpc, avx512icl); + assign_itx2_bpc_fn! (c, R, 32, 16, 8 bpc, avx512icl); + assign_itx2_bpc_fn! (c, 32, 32, 8 bpc, avx512icl); + assign_itx1_bpc_fn! (c, R, 16, 64, 8 bpc, avx512icl); + assign_itx1_bpc_fn! (c, R, 32, 64, 8 bpc, avx512icl); + assign_itx1_bpc_fn! (c, R, 64, 16, 8 bpc, avx512icl); + assign_itx1_bpc_fn! (c, R, 64, 32, 8 bpc, avx512icl); + assign_itx1_bpc_fn! (c, 64, 64, 8 bpc, avx512icl); + } else { + if bpc == 10 { + assign_itx16_bpc_fn!(c, 8, 8, 10 bpc, avx512icl); + assign_itx16_bpc_fn!(c, R, 8, 16, 10 bpc, avx512icl); + assign_itx16_bpc_fn!(c, R, 16, 8, 10 bpc, avx512icl); + assign_itx12_bpc_fn!(c, 16, 16, 10 bpc, avx512icl); + assign_itx2_bpc_fn! (c, R, 8, 32, 10 bpc, avx512icl); + assign_itx2_bpc_fn! (c, R, 16, 32, 10 bpc, avx512icl); + assign_itx2_bpc_fn! (c, R, 32, 8, 10 bpc, avx512icl); + assign_itx2_bpc_fn! (c, R, 32, 16, 10 bpc, avx512icl); + assign_itx2_bpc_fn! (c, 32, 32, 10 bpc, avx512icl); + assign_itx1_bpc_fn! (c, R, 16, 64, 10 bpc, avx512icl); + assign_itx1_bpc_fn! (c, R, 32, 64, 10 bpc, avx512icl); + assign_itx1_bpc_fn! (c, R, 64, 16, 10 bpc, avx512icl); + assign_itx1_bpc_fn! (c, R, 64, 32, 10 bpc, avx512icl); + assign_itx1_bpc_fn! (c, 64, 64, 10 bpc, avx512icl); + } + } + } +} + +#[cfg(all(feature = "asm", any(target_arch = "arm", target_arch = "aarch64")))] +#[inline(always)] +#[rustfmt::skip] +unsafe fn itx_dsp_init_arm(c: *mut Rav1dInvTxfmDSPContext, bpc: c_int) { + let flags = rav1d_get_cpu_flags(); + + if !flags.contains(CpuFlags::NEON) { + return; + } + + if BD::BITDEPTH == 16 && bpc != 10 { + return; + } + + assign_itx_fn! (c, BD, 4, 4, wht_wht, WHT_WHT, neon); + assign_itx16_fn!(c, BD, 4, 4, neon); + assign_itx16_fn!(c, BD, R, 4, 8, neon); + assign_itx16_fn!(c, BD, R, 4, 16, neon); + assign_itx16_fn!(c, BD, R, 8, 4, neon); + assign_itx16_fn!(c, BD, 8, 8, neon); + assign_itx16_fn!(c, BD, R, 8, 16, neon); + assign_itx16_fn!(c, BD, R, 16, 4, neon); + assign_itx16_fn!(c, BD, R, 16, 8, neon); + assign_itx12_fn!(c, BD, 16, 16, neon); + assign_itx2_fn! (c, BD, R, 8, 32, neon); + assign_itx2_fn! (c, BD, R, 16, 32, neon); + assign_itx2_fn! (c, BD, R, 32, 8, neon); + assign_itx2_fn! (c, BD, R, 32, 16, neon); + assign_itx2_fn! (c, BD, 32, 32, neon); + assign_itx1_fn! (c, BD, R, 16, 64, neon); + assign_itx1_fn! (c, BD, R, 32, 64, neon); + assign_itx1_fn! (c, BD, R, 64, 16, neon); + assign_itx1_fn! (c, BD, R, 64, 32, neon); + assign_itx1_fn! (c, BD, 64, 64, neon); +} + +macro_rules! assign_itx_all_fn64 { + ($c:ident, $BD:ty, $w:literal, $h:literal) => {{ + use paste::paste; + + paste! { + (*$c).itxfm_add[[] as usize][DCT_DCT as usize] = + Some([< inv_txfm_add_dct_dct_ $w x $h _c_erased >]::); + } + }}; + + ($c:ident, $BD:ty, $w:literal, $h:literal, $pfx:ident) => {{ + use paste::paste; + + paste! { + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][DCT_DCT as usize] + = Some([< inv_txfm_add_dct_dct_ $w x $h _c_erased >]::); + } + }}; +} + +macro_rules! assign_itx_all_fn32 { + ($c:ident, $BD:ty, $w:literal, $h:literal) => {{ + use paste::paste; + + assign_itx_all_fn64!($c, BD, $w, $h); + paste! { + (*$c).itxfm_add[[] as usize][IDTX as usize] + = Some([< inv_txfm_add_identity_identity_ $w x $h _c_erased >]::); + } + }}; + + ($c:ident, $BD:ty, $w:literal, $h:literal, $pfx:ident) => {{ + use paste::paste; + + assign_itx_all_fn64!($c, BD, $w, $h, $pfx); + paste! { + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][IDTX as usize] + = Some([< inv_txfm_add_identity_identity_ $w x $h _c_erased >]::); + } + }}; +} + +macro_rules! assign_itx_all_fn16 { + ($c:ident, $BD:ty, $w:literal, $h:literal) => {{ + use paste::paste; + + assign_itx_all_fn32!($c, BD, $w, $h); + paste! { + (*$c).itxfm_add[[] as usize][DCT_ADST as usize] + = Some([< inv_txfm_add_adst_dct_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[] as usize][ADST_DCT as usize] + = Some([< inv_txfm_add_dct_adst_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[] as usize][ADST_ADST as usize] + = Some([< inv_txfm_add_adst_adst_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[] as usize][ADST_FLIPADST as usize] + = Some([< inv_txfm_add_flipadst_adst_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[] as usize][FLIPADST_ADST as usize] + = Some([< inv_txfm_add_adst_flipadst_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[] as usize][DCT_FLIPADST as usize] + = Some([< inv_txfm_add_flipadst_dct_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[] as usize][FLIPADST_DCT as usize] + = Some([< inv_txfm_add_dct_flipadst_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[] as usize][FLIPADST_FLIPADST as usize] + = Some([< inv_txfm_add_flipadst_flipadst_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[] as usize][H_DCT as usize] + = Some([< inv_txfm_add_dct_identity_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[] as usize][V_DCT as usize] + = Some([< inv_txfm_add_identity_dct_ $w x $h _c_erased >]::); + } + }}; + + ($c:ident, $BD:ty, $w:literal, $h:literal, $pfx:ident) => {{ + use paste::paste; + + assign_itx_all_fn32!($c, BD, $w, $h, $pfx); + paste! { + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][DCT_ADST as usize] + = Some([< inv_txfm_add_adst_dct_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][ADST_DCT as usize] + = Some([< inv_txfm_add_dct_adst_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][ADST_ADST as usize] + = Some([< inv_txfm_add_adst_adst_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][ADST_FLIPADST as usize] + = Some([< inv_txfm_add_flipadst_adst_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][FLIPADST_ADST as usize] + = Some([< inv_txfm_add_adst_flipadst_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][DCT_FLIPADST as usize] + = Some([< inv_txfm_add_flipadst_dct_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][FLIPADST_DCT as usize] + = Some([< inv_txfm_add_dct_flipadst_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][FLIPADST_FLIPADST as usize] + = Some([< inv_txfm_add_flipadst_flipadst_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][H_DCT as usize] + = Some([< inv_txfm_add_dct_identity_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][V_DCT as usize] + = Some([< inv_txfm_add_identity_dct_ $w x $h _c_erased >]::); + } + }}; +} + +macro_rules! assign_itx_all_fn84 { + ($c:ident, $BD:ty, $w:literal, $h:literal) => {{ + use paste::paste; + + assign_itx_all_fn16!($c, BD, $w, $h); + paste! { + (*$c).itxfm_add[[] as usize][H_FLIPADST as usize] + = Some([< inv_txfm_add_flipadst_identity_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[] as usize][V_FLIPADST as usize] + = Some([< inv_txfm_add_identity_flipadst_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[] as usize][H_ADST as usize] + = Some([< inv_txfm_add_adst_identity_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[] as usize][V_ADST as usize] + = Some([< inv_txfm_add_identity_adst_ $w x $h _c_erased >]::); + } + }}; + + ($c:ident, $BD:ty, $w:literal, $h:literal, $pfx:ident) => {{ + use paste::paste; + + assign_itx_all_fn16!($c, BD, $w, $h, $pfx); + paste! { + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][H_FLIPADST as usize] + = Some([< inv_txfm_add_flipadst_identity_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][V_FLIPADST as usize] + = Some([< inv_txfm_add_identity_flipadst_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][H_ADST as usize] + = Some([< inv_txfm_add_adst_identity_ $w x $h _c_erased >]::); + (*$c).itxfm_add[[<$pfx TX_ $w X $h>] as usize][V_ADST as usize] + = Some([< inv_txfm_add_identity_adst_ $w x $h _c_erased >]::); + } + }}; +} + +#[cold] +#[rustfmt::skip] +pub unsafe fn rav1d_itx_dsp_init(c: *mut Rav1dInvTxfmDSPContext, mut _bpc: c_int) { + + + (*c).itxfm_add[TX_4X4 as usize][WHT_WHT as usize] + = Some(inv_txfm_add_wht_wht_4x4_c_erased::); + assign_itx_all_fn84!(c, BD, 4, 4 ); + assign_itx_all_fn84!(c, BD, 4, 8, R); + assign_itx_all_fn84!(c, BD, 4, 16, R); + assign_itx_all_fn84!(c, BD, 8, 4, R); + assign_itx_all_fn84!(c, BD, 8, 8 ); + assign_itx_all_fn84!(c, BD, 8, 16, R); + assign_itx_all_fn32!(c, BD, 8, 32, R); + assign_itx_all_fn84!(c, BD, 16, 4, R); + assign_itx_all_fn84!(c, BD, 16, 8, R); + assign_itx_all_fn16!(c, BD, 16, 16 ); + assign_itx_all_fn32!(c, BD, 16, 32, R); + assign_itx_all_fn64!(c, BD, 16, 64, R); + assign_itx_all_fn32!(c, BD, 32, 8, R); + assign_itx_all_fn32!(c, BD, 32, 16, R); + assign_itx_all_fn32!(c, BD, 32, 32 ); + assign_itx_all_fn64!(c, BD, 32, 64, R); + assign_itx_all_fn64!(c, BD, 64, 16, R); + assign_itx_all_fn64!(c, BD, 64, 32, R); + assign_itx_all_fn64!(c, BD, 64, 64 ); + + #[cfg(feature = "asm")] + cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + itx_dsp_init_x86::(c, _bpc); + } else if #[cfg(any(target_arch = "arm", target_arch = "aarch64"))] { + itx_dsp_init_arm::(c, _bpc); + } + } +} diff --git a/src/itx_tmpl_16.rs b/src/itx_tmpl_16.rs deleted file mode 100644 index 2fb34d510..000000000 --- a/src/itx_tmpl_16.rs +++ /dev/null @@ -1,1366 +0,0 @@ -use crate::include::common::bitdepth::BitDepth16; -use crate::include::common::bitdepth::DynCoef; -use crate::include::common::bitdepth::DynPixel; -use crate::include::common::intops::iclip; -use crate::src::itx::Rav1dInvTxfmDSPContext; -use crate::src::levels::ADST_ADST; -use crate::src::levels::ADST_DCT; -use crate::src::levels::ADST_FLIPADST; -use crate::src::levels::DCT_ADST; -use crate::src::levels::DCT_DCT; -use crate::src::levels::DCT_FLIPADST; -use crate::src::levels::FLIPADST_ADST; -use crate::src::levels::FLIPADST_DCT; -use crate::src::levels::FLIPADST_FLIPADST; -use crate::src::levels::H_ADST; -use crate::src::levels::H_DCT; -use crate::src::levels::H_FLIPADST; -use crate::src::levels::IDTX; -use crate::src::levels::RTX_16X32; -use crate::src::levels::RTX_16X4; -use crate::src::levels::RTX_16X64; -use crate::src::levels::RTX_16X8; -use crate::src::levels::RTX_32X16; -use crate::src::levels::RTX_32X64; -use crate::src::levels::RTX_32X8; -use crate::src::levels::RTX_4X16; -use crate::src::levels::RTX_4X8; -use crate::src::levels::RTX_64X16; -use crate::src::levels::RTX_64X32; -use crate::src::levels::RTX_8X16; -use crate::src::levels::RTX_8X32; -use crate::src::levels::RTX_8X4; -use crate::src::levels::TX_16X16; -use crate::src::levels::TX_32X32; -use crate::src::levels::TX_4X4; -use crate::src::levels::TX_64X64; -use crate::src::levels::TX_8X8; -use crate::src::levels::V_ADST; -use crate::src::levels::V_DCT; -use crate::src::levels::V_FLIPADST; -use crate::src::levels::WHT_WHT; -use libc::memset; -use libc::ptrdiff_t; -use std::ffi::c_int; -use std::ffi::c_void; - -#[cfg(feature = "asm")] -use crate::src::cpu::{rav1d_get_cpu_flags, CpuFlags}; - -#[cfg(feature = "asm")] -use cfg_if::cfg_if; - -pub type pixel = u16; -pub type coef = i32; - -#[inline] -unsafe fn PXSTRIDE(x: ptrdiff_t) -> ptrdiff_t { - if x & 1 != 0 { - unreachable!(); - } - return x >> 1; -} - -unsafe extern "C" fn inv_txfm_add_wht_wht_4x4_c_erased( - dst: *mut DynPixel, - stride: ptrdiff_t, - coeff: *mut DynCoef, - eob: c_int, - bitdepth_max: c_int, -) { - inv_txfm_add_wht_wht_4x4_rust(dst.cast(), stride, coeff.cast(), eob, bitdepth_max); -} - -unsafe fn inv_txfm_add_wht_wht_4x4_rust( - mut dst: *mut pixel, - stride: ptrdiff_t, - coeff: *mut coef, - _eob: c_int, - bitdepth_max: c_int, -) { - use crate::src::itx_1d::dav1d_inv_wht4_1d_c; - - let mut tmp: [i32; 16] = [0; 16]; - let mut c: *mut i32 = tmp.as_mut_ptr(); - let mut y = 0; - while y < 4 { - let mut x = 0; - while x < 4 { - *c.offset(x as isize) = *coeff.offset((y + x * 4) as isize) >> 2; - x += 1; - } - dav1d_inv_wht4_1d_c(c, 1 as c_int as ptrdiff_t); - y += 1; - c = c.offset(4); - } - memset( - coeff as *mut c_void, - 0 as c_int, - ::core::mem::size_of::() - .wrapping_mul(4) - .wrapping_mul(4), - ); - let mut x_0 = 0; - while x_0 < 4 { - dav1d_inv_wht4_1d_c( - &mut *tmp.as_mut_ptr().offset(x_0 as isize), - 4 as c_int as ptrdiff_t, - ); - x_0 += 1; - } - c = tmp.as_mut_ptr(); - let mut y_0 = 0; - while y_0 < 4 { - let mut x_1 = 0; - while x_1 < 4 { - let fresh1 = c; - c = c.offset(1); - *dst.offset(x_1 as isize) = iclip( - *dst.offset(x_1 as isize) as c_int + *fresh1, - 0 as c_int, - bitdepth_max, - ) as pixel; - x_1 += 1; - } - y_0 += 1; - dst = dst.offset(PXSTRIDE(stride) as isize); - } -} - -#[cfg(all(feature = "asm", any(target_arch = "x86", target_arch = "x86_64")))] -#[inline(always)] -#[rustfmt::skip] -unsafe fn itx_dsp_init_x86(c: *mut Rav1dInvTxfmDSPContext, bpc: c_int) { - // TODO(legare): Temporary import until init fns are deduplicated. - use crate::src::itx::*; - - let flags = rav1d_get_cpu_flags(); - - if !flags.contains(CpuFlags::SSE2) { - return; - } - - (*c).itxfm_add[TX_4X4 as usize][WHT_WHT as usize] = Some(dav1d_inv_txfm_add_wht_wht_4x4_16bpc_sse2); - - if !flags.contains(CpuFlags::SSSE3) { - return; - } - - if !flags.contains(CpuFlags::SSE41) { - return; - } - - if bpc == 10 { - (*c).itxfm_add[TX_4X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x4_16bpc_sse4); - (*c).itxfm_add[TX_4X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x4_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x8_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_4X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x4_16bpc_sse4); - (*c).itxfm_add[RTX_8X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x4_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x8_16bpc_sse4); - (*c).itxfm_add[TX_8X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x8_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x16_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x4_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x8_16bpc_sse4); - (*c).itxfm_add[TX_16X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x16_16bpc_sse4); - (*c).itxfm_add[TX_16X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x16_16bpc_sse4); - (*c).itxfm_add[TX_16X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x16_16bpc_sse4); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x16_16bpc_sse4); - (*c).itxfm_add[TX_16X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x16_16bpc_sse4); - (*c).itxfm_add[TX_16X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x16_16bpc_sse4); - (*c).itxfm_add[TX_16X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x16_16bpc_sse4); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x16_16bpc_sse4); - (*c).itxfm_add[TX_16X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x16_16bpc_sse4); - (*c).itxfm_add[TX_16X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x16_16bpc_sse4); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x16_16bpc_sse4); - (*c).itxfm_add[TX_16X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x16_16bpc_sse4); - (*c).itxfm_add[RTX_8X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x32_16bpc_sse4); - (*c).itxfm_add[RTX_8X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x32_16bpc_sse4); - (*c).itxfm_add[RTX_32X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x8_16bpc_sse4); - (*c).itxfm_add[RTX_32X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x8_16bpc_sse4); - (*c).itxfm_add[RTX_16X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x32_16bpc_sse4); - (*c).itxfm_add[RTX_16X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x32_16bpc_sse4); - (*c).itxfm_add[RTX_32X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x16_16bpc_sse4); - (*c).itxfm_add[RTX_32X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x16_16bpc_sse4); - (*c).itxfm_add[TX_32X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x32_16bpc_sse4); - (*c).itxfm_add[TX_32X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x32_16bpc_sse4); - (*c).itxfm_add[RTX_16X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x64_16bpc_sse4); - (*c).itxfm_add[RTX_32X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x64_16bpc_sse4); - (*c).itxfm_add[RTX_64X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x16_16bpc_sse4); - (*c).itxfm_add[RTX_64X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x32_16bpc_sse4); - (*c).itxfm_add[TX_64X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x64_16bpc_sse4); - } - - - #[cfg(target_arch = "x86_64")] - { - if !flags.contains(CpuFlags::AVX2) { - return; - } - - (*c).itxfm_add[TX_4X4 as usize][WHT_WHT as usize] = Some(dav1d_inv_txfm_add_wht_wht_4x4_16bpc_avx2); - - if bpc == 10 { - (*c).itxfm_add[TX_4X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x4_10bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x4_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x8_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x4_10bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x4_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x8_10bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x8_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x16_10bpc_avx2); - (*c).itxfm_add[RTX_8X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x32_10bpc_avx2); - (*c).itxfm_add[RTX_8X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x32_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x4_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x8_10bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x8_10bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x16_10bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x16_10bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x16_10bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x16_10bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x16_10bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x16_10bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x16_10bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x16_10bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x16_10bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x16_10bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x16_10bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x16_10bpc_avx2); - (*c).itxfm_add[RTX_16X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x32_10bpc_avx2); - (*c).itxfm_add[RTX_16X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x32_10bpc_avx2); - (*c).itxfm_add[RTX_16X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x64_10bpc_avx2); - (*c).itxfm_add[RTX_32X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x8_10bpc_avx2); - (*c).itxfm_add[RTX_32X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x8_10bpc_avx2); - (*c).itxfm_add[RTX_32X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x16_10bpc_avx2); - (*c).itxfm_add[RTX_32X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x16_10bpc_avx2); - (*c).itxfm_add[TX_32X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x32_10bpc_avx2); - (*c).itxfm_add[TX_32X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x32_10bpc_avx2); - (*c).itxfm_add[RTX_32X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x64_10bpc_avx2); - (*c).itxfm_add[RTX_64X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x16_10bpc_avx2); - (*c).itxfm_add[RTX_64X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x32_10bpc_avx2); - (*c).itxfm_add[TX_64X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x64_10bpc_avx2); - } else { - (*c).itxfm_add[TX_4X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x4_12bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x4_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x8_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x4_12bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x4_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x8_12bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x8_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x16_12bpc_avx2); - (*c).itxfm_add[RTX_8X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x32_12bpc_avx2); - (*c).itxfm_add[RTX_8X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x32_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x4_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x8_12bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x16_12bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x16_12bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x16_12bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x16_12bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x16_12bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x16_12bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x16_12bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x16_12bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x16_12bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x16_12bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x16_12bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x16_12bpc_avx2); - (*c).itxfm_add[RTX_32X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x8_12bpc_avx2); - (*c).itxfm_add[RTX_32X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x8_12bpc_avx2); - (*c).itxfm_add[RTX_16X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x32_12bpc_avx2); - (*c).itxfm_add[RTX_32X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x16_12bpc_avx2); - (*c).itxfm_add[TX_32X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x32_12bpc_avx2); - } - - if !flags.contains(CpuFlags::AVX512ICL) { - return; - } - - if bpc == 10 { - (*c).itxfm_add[TX_8X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x8_10bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x32_10bpc_avx512icl); - (*c).itxfm_add[RTX_8X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x32_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x8_10bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x16_10bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x16_10bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x16_10bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x16_10bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x16_10bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x16_10bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x16_10bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x16_10bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x16_10bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x16_10bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x16_10bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x32_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x32_10bpc_avx512icl); - (*c).itxfm_add[RTX_32X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_32X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x8_10bpc_avx512icl); - (*c).itxfm_add[RTX_32X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_32X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x16_10bpc_avx512icl); - (*c).itxfm_add[TX_32X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x32_10bpc_avx512icl); - (*c).itxfm_add[TX_32X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x32_10bpc_avx512icl); - (*c).itxfm_add[RTX_16X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x64_10bpc_avx512icl); - (*c).itxfm_add[RTX_32X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x64_10bpc_avx512icl); - (*c).itxfm_add[RTX_64X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x16_10bpc_avx512icl); - (*c).itxfm_add[RTX_64X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x32_10bpc_avx512icl); - (*c).itxfm_add[TX_64X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x64_10bpc_avx512icl); - } - } -} - -#[cfg(all(feature = "asm", any(target_arch = "arm", target_arch = "aarch64")))] -#[inline(always)] -unsafe fn itx_dsp_init_arm(c: *mut Rav1dInvTxfmDSPContext, bpc: c_int) { - // TODO(legare): Temporary import until init fns are deduplicated. - use crate::src::itx::*; - - let flags = rav1d_get_cpu_flags(); - - if !flags.contains(CpuFlags::NEON) { - return; - } - - if bpc != 10 { - return; - } - - (*c).itxfm_add[TX_4X4 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][IDTX as usize] = - Some(dav1d_inv_txfm_add_identity_identity_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][ADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_adst_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_flipadst_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][H_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_identity_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][DCT_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_dct_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][ADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_adst_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_flipadst_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][DCT_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_dct_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][ADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_adst_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_flipadst_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][V_DCT as usize] = - Some(dav1d_inv_txfm_add_identity_dct_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][H_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_identity_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][H_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_identity_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][V_ADST as usize] = - Some(dav1d_inv_txfm_add_identity_adst_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][V_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_identity_flipadst_4x4_16bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][WHT_WHT as usize] = - Some(dav1d_inv_txfm_add_wht_wht_4x4_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][IDTX as usize] = - Some(dav1d_inv_txfm_add_identity_identity_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][ADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_adst_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_flipadst_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][H_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_identity_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][DCT_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_dct_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][ADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_adst_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_flipadst_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][DCT_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_dct_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][ADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_adst_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_flipadst_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][V_DCT as usize] = - Some(dav1d_inv_txfm_add_identity_dct_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][H_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_identity_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][H_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_identity_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][V_ADST as usize] = - Some(dav1d_inv_txfm_add_identity_adst_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][V_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_identity_flipadst_4x8_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][IDTX as usize] = - Some(dav1d_inv_txfm_add_identity_identity_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][ADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_adst_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_flipadst_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][H_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_identity_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][DCT_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_dct_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][ADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_adst_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_flipadst_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][DCT_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_dct_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][ADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_adst_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_flipadst_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][V_DCT as usize] = - Some(dav1d_inv_txfm_add_identity_dct_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][H_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_identity_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][H_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_identity_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][V_ADST as usize] = - Some(dav1d_inv_txfm_add_identity_adst_4x16_16bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][V_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_identity_flipadst_4x16_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][IDTX as usize] = - Some(dav1d_inv_txfm_add_identity_identity_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][ADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_adst_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_flipadst_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][H_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_identity_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][DCT_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_dct_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][ADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_adst_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_flipadst_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][DCT_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_dct_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][ADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_adst_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_flipadst_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][V_DCT as usize] = - Some(dav1d_inv_txfm_add_identity_dct_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][H_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_identity_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][H_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_identity_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][V_ADST as usize] = - Some(dav1d_inv_txfm_add_identity_adst_8x4_16bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][V_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_identity_flipadst_8x4_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][IDTX as usize] = - Some(dav1d_inv_txfm_add_identity_identity_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][ADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_adst_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_flipadst_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][H_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_identity_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][DCT_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_dct_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][ADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_adst_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_flipadst_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][DCT_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_dct_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][ADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_adst_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_flipadst_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][V_DCT as usize] = - Some(dav1d_inv_txfm_add_identity_dct_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][H_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_identity_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][H_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_identity_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][V_ADST as usize] = - Some(dav1d_inv_txfm_add_identity_adst_8x8_16bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][V_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_identity_flipadst_8x8_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][IDTX as usize] = - Some(dav1d_inv_txfm_add_identity_identity_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][ADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_adst_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_flipadst_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][H_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_identity_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][DCT_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_dct_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][ADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_adst_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_flipadst_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][DCT_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_dct_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][ADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_adst_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_flipadst_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][V_DCT as usize] = - Some(dav1d_inv_txfm_add_identity_dct_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][H_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_identity_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][H_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_identity_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][V_ADST as usize] = - Some(dav1d_inv_txfm_add_identity_adst_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][V_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_identity_flipadst_8x16_16bpc_neon); - (*c).itxfm_add[RTX_8X32 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_8x32_16bpc_neon); - (*c).itxfm_add[RTX_8X32 as usize][IDTX as usize] = - Some(dav1d_inv_txfm_add_identity_identity_8x32_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][IDTX as usize] = - Some(dav1d_inv_txfm_add_identity_identity_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][ADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_adst_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_flipadst_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][H_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_identity_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][DCT_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_dct_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][ADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_adst_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_flipadst_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][DCT_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_dct_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][ADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_adst_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_flipadst_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][V_DCT as usize] = - Some(dav1d_inv_txfm_add_identity_dct_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][H_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_identity_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][H_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_identity_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][V_ADST as usize] = - Some(dav1d_inv_txfm_add_identity_adst_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][V_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_identity_flipadst_16x4_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][IDTX as usize] = - Some(dav1d_inv_txfm_add_identity_identity_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][ADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_adst_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_flipadst_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][H_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_identity_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][DCT_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_dct_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][ADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_adst_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_flipadst_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][DCT_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_dct_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][ADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_adst_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_flipadst_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][V_DCT as usize] = - Some(dav1d_inv_txfm_add_identity_dct_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][H_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_identity_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][H_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_identity_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][V_ADST as usize] = - Some(dav1d_inv_txfm_add_identity_adst_16x8_16bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][V_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_identity_flipadst_16x8_16bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_16x16_16bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][IDTX as usize] = - Some(dav1d_inv_txfm_add_identity_identity_16x16_16bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][ADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_adst_16x16_16bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_flipadst_16x16_16bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][H_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_identity_16x16_16bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][DCT_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_dct_16x16_16bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][ADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_adst_16x16_16bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_ADST as usize] = - Some(dav1d_inv_txfm_add_adst_flipadst_16x16_16bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][DCT_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_dct_16x16_16bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][ADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_adst_16x16_16bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_FLIPADST as usize] = - Some(dav1d_inv_txfm_add_flipadst_flipadst_16x16_16bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][V_DCT as usize] = - Some(dav1d_inv_txfm_add_identity_dct_16x16_16bpc_neon); - (*c).itxfm_add[RTX_16X32 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_16x32_16bpc_neon); - (*c).itxfm_add[RTX_16X32 as usize][IDTX as usize] = - Some(dav1d_inv_txfm_add_identity_identity_16x32_16bpc_neon); - (*c).itxfm_add[RTX_16X64 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_16x64_16bpc_neon); - (*c).itxfm_add[RTX_32X8 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_32x8_16bpc_neon); - (*c).itxfm_add[RTX_32X8 as usize][IDTX as usize] = - Some(dav1d_inv_txfm_add_identity_identity_32x8_16bpc_neon); - (*c).itxfm_add[RTX_32X16 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_32x16_16bpc_neon); - (*c).itxfm_add[RTX_32X16 as usize][IDTX as usize] = - Some(dav1d_inv_txfm_add_identity_identity_32x16_16bpc_neon); - (*c).itxfm_add[TX_32X32 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_32x32_16bpc_neon); - (*c).itxfm_add[TX_32X32 as usize][IDTX as usize] = - Some(dav1d_inv_txfm_add_identity_identity_32x32_16bpc_neon); - (*c).itxfm_add[RTX_32X64 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_32x64_16bpc_neon); - (*c).itxfm_add[RTX_64X16 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_64x16_16bpc_neon); - (*c).itxfm_add[RTX_64X32 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_64x32_16bpc_neon); - (*c).itxfm_add[TX_64X64 as usize][DCT_DCT as usize] = - Some(dav1d_inv_txfm_add_dct_dct_64x64_16bpc_neon); -} - -#[cold] -#[rustfmt::skip] -pub unsafe fn rav1d_itx_dsp_init_16bpc(c: *mut Rav1dInvTxfmDSPContext, mut _bpc: c_int) { - // TODO(legare): Temporary import until init fns are deduplicated. - use crate::src::itx::*; - - (*c).itxfm_add[TX_4X4 as usize][WHT_WHT as usize] = Some(inv_txfm_add_wht_wht_4x4_c_erased); - (*c).itxfm_add[TX_4X4 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_4x4_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_4x16_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_8x4_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_8x8_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X32 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_8x32_c_erased::); - (*c).itxfm_add[RTX_8X32 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_8x32_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_16x8_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_16x16_c_erased::); - (*c).itxfm_add[RTX_16X32 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_16x32_c_erased::); - (*c).itxfm_add[RTX_16X32 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_16x32_c_erased::); - (*c).itxfm_add[RTX_16X64 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_16x64_c_erased::); - (*c).itxfm_add[RTX_32X8 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_32x8_c_erased::); - (*c).itxfm_add[RTX_32X8 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_32x8_c_erased::); - (*c).itxfm_add[RTX_32X16 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_32x16_c_erased::); - (*c).itxfm_add[RTX_32X16 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_32x16_c_erased::); - (*c).itxfm_add[TX_32X32 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_32x32_c_erased::); - (*c).itxfm_add[TX_32X32 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_32x32_c_erased::); - (*c).itxfm_add[RTX_32X64 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_32x64_c_erased::); - (*c).itxfm_add[RTX_64X16 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_64x16_c_erased::); - (*c).itxfm_add[RTX_64X32 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_64x32_c_erased::); - (*c).itxfm_add[TX_64X64 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_64x64_c_erased::); - - #[cfg(feature = "asm")] - cfg_if! { - if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { - itx_dsp_init_x86(c, _bpc); - } else if #[cfg(any(target_arch = "arm", target_arch = "aarch64"))] { - itx_dsp_init_arm(c, _bpc); - } - } -} diff --git a/src/itx_tmpl_8.rs b/src/itx_tmpl_8.rs deleted file mode 100644 index 7d5e7f8d7..000000000 --- a/src/itx_tmpl_8.rs +++ /dev/null @@ -1,1119 +0,0 @@ -use crate::include::common::bitdepth::BitDepth8; -use crate::include::common::bitdepth::DynCoef; -use crate::include::common::bitdepth::DynPixel; -use crate::include::common::intops::iclip_u8; -use crate::src::itx::Rav1dInvTxfmDSPContext; -use crate::src::levels::ADST_ADST; -use crate::src::levels::ADST_DCT; -use crate::src::levels::ADST_FLIPADST; -use crate::src::levels::DCT_ADST; -use crate::src::levels::DCT_DCT; -use crate::src::levels::DCT_FLIPADST; -use crate::src::levels::FLIPADST_ADST; -use crate::src::levels::FLIPADST_DCT; -use crate::src::levels::FLIPADST_FLIPADST; -use crate::src::levels::H_ADST; -use crate::src::levels::H_DCT; -use crate::src::levels::H_FLIPADST; -use crate::src::levels::IDTX; -use crate::src::levels::RTX_16X32; -use crate::src::levels::RTX_16X4; -use crate::src::levels::RTX_16X64; -use crate::src::levels::RTX_16X8; -use crate::src::levels::RTX_32X16; -use crate::src::levels::RTX_32X64; -use crate::src::levels::RTX_32X8; -use crate::src::levels::RTX_4X16; -use crate::src::levels::RTX_4X8; -use crate::src::levels::RTX_64X16; -use crate::src::levels::RTX_64X32; -use crate::src::levels::RTX_8X16; -use crate::src::levels::RTX_8X32; -use crate::src::levels::RTX_8X4; -use crate::src::levels::TX_16X16; -use crate::src::levels::TX_32X32; -use crate::src::levels::TX_4X4; -use crate::src::levels::TX_64X64; -use crate::src::levels::TX_8X8; -use crate::src::levels::V_ADST; -use crate::src::levels::V_DCT; -use crate::src::levels::V_FLIPADST; -use crate::src::levels::WHT_WHT; -use libc::memset; -use libc::ptrdiff_t; -use std::ffi::c_int; -use std::ffi::c_void; - -#[cfg(feature = "asm")] -use crate::src::cpu::{rav1d_get_cpu_flags, CpuFlags}; - -#[cfg(feature = "asm")] -use cfg_if::cfg_if; - -pub type pixel = u8; -pub type coef = i16; - -unsafe extern "C" fn inv_txfm_add_wht_wht_4x4_c_erased( - dst: *mut DynPixel, - stride: ptrdiff_t, - coeff: *mut DynCoef, - eob: c_int, - _bitdepth_max: c_int, -) { - inv_txfm_add_wht_wht_4x4_rust(dst.cast(), stride, coeff.cast(), eob); -} - -unsafe fn inv_txfm_add_wht_wht_4x4_rust( - mut dst: *mut pixel, - stride: ptrdiff_t, - coeff: *mut coef, - _eob: c_int, -) { - use crate::src::itx_1d::dav1d_inv_wht4_1d_c; - - let mut tmp: [i32; 16] = [0; 16]; - let mut c: *mut i32 = tmp.as_mut_ptr(); - let mut y = 0; - while y < 4 { - let mut x = 0; - while x < 4 { - *c.offset(x as isize) = *coeff.offset((y + x * 4) as isize) as c_int >> 2; - x += 1; - } - dav1d_inv_wht4_1d_c(c, 1 as c_int as ptrdiff_t); - y += 1; - c = c.offset(4); - } - memset( - coeff as *mut c_void, - 0 as c_int, - ::core::mem::size_of::() - .wrapping_mul(4) - .wrapping_mul(4), - ); - let mut x_0 = 0; - while x_0 < 4 { - dav1d_inv_wht4_1d_c( - &mut *tmp.as_mut_ptr().offset(x_0 as isize), - 4 as c_int as ptrdiff_t, - ); - x_0 += 1; - } - c = tmp.as_mut_ptr(); - let mut y_0 = 0; - while y_0 < 4 { - let mut x_1 = 0; - while x_1 < 4 { - let fresh1 = c; - c = c.offset(1); - *dst.offset(x_1 as isize) = - iclip_u8(*dst.offset(x_1 as isize) as c_int + *fresh1) as pixel; - x_1 += 1; - } - y_0 += 1; - dst = dst.offset(stride as isize); - } -} - -#[cfg(all(feature = "asm", any(target_arch = "x86", target_arch = "x86_64")))] -#[inline(always)] -#[rustfmt::skip] -unsafe fn itx_dsp_init_x86(c: *mut Rav1dInvTxfmDSPContext, _bpc: c_int) { - // TODO(legare): Temporary import until init fns are deduplicated. - use crate::src::itx::*; - - let flags = rav1d_get_cpu_flags(); - - if !flags.contains(CpuFlags::SSE2) { - return; - } - - (*c).itxfm_add[TX_4X4 as usize][WHT_WHT as usize] = Some(dav1d_inv_txfm_add_wht_wht_4x4_8bpc_sse2); - - if !flags.contains(CpuFlags::SSSE3) { - return; - } - - (*c).itxfm_add[TX_4X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x4_8bpc_ssse3); - (*c).itxfm_add[TX_4X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x4_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x8_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x4_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x8_8bpc_ssse3); - (*c).itxfm_add[TX_8X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x8_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_4X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x16_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_16X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x4_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x16_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x8_8bpc_ssse3); - (*c).itxfm_add[TX_16X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x16_8bpc_ssse3); - (*c).itxfm_add[TX_16X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x16_8bpc_ssse3); - (*c).itxfm_add[TX_16X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x16_8bpc_ssse3); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x16_8bpc_ssse3); - (*c).itxfm_add[TX_16X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x16_8bpc_ssse3); - (*c).itxfm_add[TX_16X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x16_8bpc_ssse3); - (*c).itxfm_add[TX_16X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x16_8bpc_ssse3); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x16_8bpc_ssse3); - (*c).itxfm_add[TX_16X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x16_8bpc_ssse3); - (*c).itxfm_add[TX_16X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x16_8bpc_ssse3); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x16_8bpc_ssse3); - (*c).itxfm_add[TX_16X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x16_8bpc_ssse3); - (*c).itxfm_add[RTX_8X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x32_8bpc_ssse3); - (*c).itxfm_add[RTX_8X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x32_8bpc_ssse3); - (*c).itxfm_add[RTX_32X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x8_8bpc_ssse3); - (*c).itxfm_add[RTX_32X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x8_8bpc_ssse3); - (*c).itxfm_add[RTX_16X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x32_8bpc_ssse3); - (*c).itxfm_add[RTX_16X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x32_8bpc_ssse3); - (*c).itxfm_add[RTX_32X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x16_8bpc_ssse3); - (*c).itxfm_add[RTX_32X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x16_8bpc_ssse3); - (*c).itxfm_add[TX_32X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x32_8bpc_ssse3); - (*c).itxfm_add[TX_32X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x32_8bpc_ssse3); - (*c).itxfm_add[RTX_16X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x64_8bpc_ssse3); - (*c).itxfm_add[RTX_32X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x64_8bpc_ssse3); - (*c).itxfm_add[RTX_64X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x16_8bpc_ssse3); - (*c).itxfm_add[RTX_64X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x32_8bpc_ssse3); - (*c).itxfm_add[TX_64X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x64_8bpc_ssse3); - - if !flags.contains(CpuFlags::SSE41) { - return; - } - - #[cfg(target_arch = "x86_64")] - { - if !flags.contains(CpuFlags::AVX2) { - return; - } - - (*c).itxfm_add[TX_4X4 as usize][WHT_WHT as usize] = Some(dav1d_inv_txfm_add_wht_wht_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x4_8bpc_avx2); - (*c).itxfm_add[TX_4X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x4_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x8_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_4X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x4_8bpc_avx2); - (*c).itxfm_add[RTX_8X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x4_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x8_8bpc_avx2); - (*c).itxfm_add[TX_8X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x8_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x16_8bpc_avx2); - (*c).itxfm_add[RTX_8X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x32_8bpc_avx2); - (*c).itxfm_add[RTX_8X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x32_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x4_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x8_8bpc_avx2); - (*c).itxfm_add[RTX_16X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x8_8bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x16_8bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x16_8bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x16_8bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x16_8bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x16_8bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x16_8bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x16_8bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x16_8bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x16_8bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x16_8bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x16_8bpc_avx2); - (*c).itxfm_add[TX_16X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x16_8bpc_avx2); - (*c).itxfm_add[RTX_16X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x32_8bpc_avx2); - (*c).itxfm_add[RTX_16X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x32_8bpc_avx2); - (*c).itxfm_add[RTX_16X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x64_8bpc_avx2); - (*c).itxfm_add[RTX_32X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x8_8bpc_avx2); - (*c).itxfm_add[RTX_32X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x8_8bpc_avx2); - (*c).itxfm_add[RTX_32X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x16_8bpc_avx2); - (*c).itxfm_add[RTX_32X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x16_8bpc_avx2); - (*c).itxfm_add[TX_32X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x32_8bpc_avx2); - (*c).itxfm_add[TX_32X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x32_8bpc_avx2); - (*c).itxfm_add[RTX_32X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x64_8bpc_avx2); - (*c).itxfm_add[RTX_64X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x16_8bpc_avx2); - (*c).itxfm_add[RTX_64X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x32_8bpc_avx2); - (*c).itxfm_add[TX_64X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x64_8bpc_avx2); - - if !flags.contains(CpuFlags::AVX512ICL) { - return; - } - - (*c).itxfm_add[TX_4X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x4_8bpc_avx512icl); - (*c).itxfm_add[TX_4X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_4X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x4_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x8_8bpc_avx512icl); - (*c).itxfm_add[TX_8X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x32_8bpc_avx512icl); - (*c).itxfm_add[RTX_8X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x32_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x4_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x8_8bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x16_8bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x16_8bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x16_8bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x16_8bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x16_8bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x16_8bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x16_8bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x16_8bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x16_8bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x16_8bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x16_8bpc_avx512icl); - (*c).itxfm_add[TX_16X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x32_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x32_8bpc_avx512icl); - (*c).itxfm_add[RTX_16X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x64_8bpc_avx512icl); - (*c).itxfm_add[RTX_32X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_32X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x8_8bpc_avx512icl); - (*c).itxfm_add[RTX_32X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_32X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x16_8bpc_avx512icl); - (*c).itxfm_add[TX_32X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x32_8bpc_avx512icl); - (*c).itxfm_add[TX_32X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x32_8bpc_avx512icl); - (*c).itxfm_add[RTX_32X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x64_8bpc_avx512icl); - (*c).itxfm_add[RTX_64X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x16_8bpc_avx512icl); - (*c).itxfm_add[RTX_64X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x32_8bpc_avx512icl); - (*c).itxfm_add[TX_64X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x64_8bpc_avx512icl); - } -} - -#[cfg(all(feature = "asm", any(target_arch = "arm", target_arch = "aarch64")))] -#[inline(always)] -#[rustfmt::skip] -unsafe fn itx_dsp_init_arm(c: *mut Rav1dInvTxfmDSPContext, mut _bpc: c_int) { - // TODO(legare): Temporary import until init fns are deduplicated. - use crate::src::itx::*; - - let flags = rav1d_get_cpu_flags(); - - if !flags.contains(CpuFlags::NEON) { - return; - } - - (*c).itxfm_add[TX_4X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x4_8bpc_neon); - (*c).itxfm_add[TX_4X4 as usize][WHT_WHT as usize] = Some(dav1d_inv_txfm_add_wht_wht_4x4_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x8_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_4x16_8bpc_neon); - (*c).itxfm_add[RTX_4X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_4x16_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x4_8bpc_neon); - (*c).itxfm_add[RTX_8X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x4_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x8_8bpc_neon); - (*c).itxfm_add[TX_8X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x8_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X16 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_8x16_8bpc_neon); - (*c).itxfm_add[RTX_8X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_8x32_8bpc_neon); - (*c).itxfm_add[RTX_8X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_8x32_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X4 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x4_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][H_ADST as usize] = Some(dav1d_inv_txfm_add_adst_identity_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][H_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_identity_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][V_ADST as usize] = Some(dav1d_inv_txfm_add_identity_adst_16x8_8bpc_neon); - (*c).itxfm_add[RTX_16X8 as usize][V_FLIPADST as usize] = Some(dav1d_inv_txfm_add_identity_flipadst_16x8_8bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x16_8bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x16_8bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][ADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_adst_16x16_8bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_DCT as usize] = Some(dav1d_inv_txfm_add_dct_flipadst_16x16_8bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][H_DCT as usize] = Some(dav1d_inv_txfm_add_dct_identity_16x16_8bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][DCT_ADST as usize] = Some(dav1d_inv_txfm_add_adst_dct_16x16_8bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][ADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_adst_16x16_8bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_ADST as usize] = Some(dav1d_inv_txfm_add_adst_flipadst_16x16_8bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][DCT_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_dct_16x16_8bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][ADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_adst_16x16_8bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_FLIPADST as usize] = Some(dav1d_inv_txfm_add_flipadst_flipadst_16x16_8bpc_neon); - (*c).itxfm_add[TX_16X16 as usize][V_DCT as usize] = Some(dav1d_inv_txfm_add_identity_dct_16x16_8bpc_neon); - (*c).itxfm_add[RTX_16X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x32_8bpc_neon); - (*c).itxfm_add[RTX_16X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_16x32_8bpc_neon); - (*c).itxfm_add[RTX_16X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_16x64_8bpc_neon); - (*c).itxfm_add[RTX_32X8 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x8_8bpc_neon); - (*c).itxfm_add[RTX_32X8 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x8_8bpc_neon); - (*c).itxfm_add[RTX_32X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x16_8bpc_neon); - (*c).itxfm_add[RTX_32X16 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x16_8bpc_neon); - (*c).itxfm_add[TX_32X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x32_8bpc_neon); - (*c).itxfm_add[TX_32X32 as usize][IDTX as usize] = Some(dav1d_inv_txfm_add_identity_identity_32x32_8bpc_neon); - (*c).itxfm_add[RTX_32X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_32x64_8bpc_neon); - (*c).itxfm_add[RTX_64X16 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x16_8bpc_neon); - (*c).itxfm_add[RTX_64X32 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x32_8bpc_neon); - (*c).itxfm_add[TX_64X64 as usize][DCT_DCT as usize] = Some(dav1d_inv_txfm_add_dct_dct_64x64_8bpc_neon); -} - -#[cold] -#[rustfmt::skip] -pub unsafe fn rav1d_itx_dsp_init_8bpc(c: *mut Rav1dInvTxfmDSPContext, mut _bpc: c_int) { - // TODO(legare): Temporary import until init fns are deduplicated. - use crate::src::itx::*; - - (*c).itxfm_add[TX_4X4 as usize][WHT_WHT as usize] = Some(inv_txfm_add_wht_wht_4x4_c_erased); - (*c).itxfm_add[TX_4X4 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_4x4_c_erased::); - (*c).itxfm_add[TX_4X4 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_4x4_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_4x8_c_erased::); - (*c).itxfm_add[RTX_4X8 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_4x8_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_4x16_c_erased::); - (*c).itxfm_add[RTX_4X16 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_4x16_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_8x4_c_erased::); - (*c).itxfm_add[RTX_8X4 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_8x4_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_8x8_c_erased::); - (*c).itxfm_add[TX_8X8 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_8x8_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_8x16_c_erased::); - (*c).itxfm_add[RTX_8X16 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_8x16_c_erased::); - (*c).itxfm_add[RTX_8X32 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_8x32_c_erased::); - (*c).itxfm_add[RTX_8X32 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_8x32_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_16x4_c_erased::); - (*c).itxfm_add[RTX_16X4 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_16x4_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][H_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_identity_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][V_FLIPADST as usize] = - Some(inv_txfm_add_identity_flipadst_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][H_ADST as usize] = - Some(inv_txfm_add_adst_identity_16x8_c_erased::); - (*c).itxfm_add[RTX_16X8 as usize][V_ADST as usize] = - Some(inv_txfm_add_identity_adst_16x8_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][DCT_ADST as usize] = - Some(inv_txfm_add_adst_dct_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][ADST_DCT as usize] = - Some(inv_txfm_add_dct_adst_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][ADST_ADST as usize] = - Some(inv_txfm_add_adst_adst_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][ADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_adst_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_ADST as usize] = - Some(inv_txfm_add_adst_flipadst_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][DCT_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_dct_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_DCT as usize] = - Some(inv_txfm_add_dct_flipadst_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][FLIPADST_FLIPADST as usize] = - Some(inv_txfm_add_flipadst_flipadst_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][H_DCT as usize] = - Some(inv_txfm_add_dct_identity_16x16_c_erased::); - (*c).itxfm_add[TX_16X16 as usize][V_DCT as usize] = - Some(inv_txfm_add_identity_dct_16x16_c_erased::); - (*c).itxfm_add[RTX_16X32 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_16x32_c_erased::); - (*c).itxfm_add[RTX_16X32 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_16x32_c_erased::); - (*c).itxfm_add[RTX_16X64 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_16x64_c_erased::); - (*c).itxfm_add[RTX_32X8 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_32x8_c_erased::); - (*c).itxfm_add[RTX_32X8 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_32x8_c_erased::); - (*c).itxfm_add[RTX_32X16 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_32x16_c_erased::); - (*c).itxfm_add[RTX_32X16 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_32x16_c_erased::); - (*c).itxfm_add[TX_32X32 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_32x32_c_erased::); - (*c).itxfm_add[TX_32X32 as usize][IDTX as usize] = - Some(inv_txfm_add_identity_identity_32x32_c_erased::); - (*c).itxfm_add[RTX_32X64 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_32x64_c_erased::); - (*c).itxfm_add[RTX_64X16 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_64x16_c_erased::); - (*c).itxfm_add[RTX_64X32 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_64x32_c_erased::); - (*c).itxfm_add[TX_64X64 as usize][DCT_DCT as usize] = - Some(inv_txfm_add_dct_dct_64x64_c_erased::); - - #[cfg(feature = "asm")] - cfg_if! { - if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { - itx_dsp_init_x86(c, _bpc); - } else if #[cfg(any(target_arch = "arm", target_arch = "aarch64"))] { - itx_dsp_init_arm(c, _bpc); - } - } -}