Skip to content

Commit

Permalink
x86_64: Move WHT_WHT transforms to common lookup table
Browse files Browse the repository at this point in the history
  • Loading branch information
@barrbrain
barrbrain committed Oct 23, 2023
1 parent 82fdf7f commit 7738eb0
Showing 1 changed file with 143 additions and 63 deletions.
206 changes: 143 additions & 63 deletions src/asm/x86/transform/inverse.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,6 @@ pub fn inverse_transform_add<T: Pixel>(
input: &[T::Coeff], output: &mut PlaneRegionMut<'_, T>, eob: usize,
tx_size: TxSize, tx_type: TxType, bd: usize, cpu: CpuFeatureLevel,
) {
if tx_type == TxType::WHT_WHT {
debug_assert!(tx_size == TxSize::TX_4X4);
match T::type_enum() {
PixelType::U8 => {
if let Some(func) = INV_TXFM_WHT_FN[cpu.as_index()] {
return call_inverse_func(func, input, output, eob, 4, 4, bd);
}
}
PixelType::U16 => {
if let Some(func) = INV_TXFM_WHT_HBD_FN[cpu.as_index()] {
return call_inverse_hbd_func(func, input, output, eob, 4, 4, bd);
}
}
}
}
match T::type_enum() {
PixelType::U8 => {
if let Some(func) = INV_TXFM_FNS[cpu.as_index()][tx_size][tx_type] {
Expand Down Expand Up @@ -81,44 +66,6 @@ pub fn inverse_transform_add<T: Pixel>(
rust::inverse_transform_add(input, output, eob, tx_size, tx_type, bd, cpu);
}

extern {
fn rav1e_inv_txfm_add_wht_wht_4x4_8bpc_avx2(
dst: *mut u8, dst_stride: libc::ptrdiff_t, coeff: *mut i16, eob: i32,
);
fn rav1e_inv_txfm_add_wht_wht_4x4_8bpc_sse2(
dst: *mut u8, dst_stride: libc::ptrdiff_t, coeff: *mut i16, eob: i32,
);
fn rav1e_inv_txfm_add_wht_wht_4x4_16bpc_avx2(
dst: *mut u16, dst_stride: libc::ptrdiff_t, coeff: *mut i16, eob: i32,
bitdepth_max: i32,
);
fn rav1e_inv_txfm_add_wht_wht_4x4_16bpc_sse2(
dst: *mut u16, dst_stride: libc::ptrdiff_t, coeff: *mut i16, eob: i32,
bitdepth_max: i32,
);
}

const INV_TXFM_WHT_FN_AVX2: Option<InvTxfmFunc> =
Some(rav1e_inv_txfm_add_wht_wht_4x4_8bpc_avx2 as _);
const INV_TXFM_WHT_FN_SSE2: Option<InvTxfmFunc> =
Some(rav1e_inv_txfm_add_wht_wht_4x4_8bpc_sse2 as _);
const INV_TXFM_WHT_HBD_FN_AVX2: Option<InvTxfmHBDFunc> =
Some(rav1e_inv_txfm_add_wht_wht_4x4_16bpc_avx2 as _);
const INV_TXFM_WHT_HBD_FN_SSE2: Option<InvTxfmHBDFunc> =
Some(rav1e_inv_txfm_add_wht_wht_4x4_16bpc_sse2 as _);

cpu_function_lookup_table!(
INV_TXFM_WHT_FN: [Option<InvTxfmFunc>],
default: None,
[SSE2, AVX2]
);

cpu_function_lookup_table!(
INV_TXFM_WHT_HBD_FN: [Option<InvTxfmHBDFunc>],
default: None,
[SSE2, AVX2]
);

macro_rules! decl_itx_fns {
// Takes a 2d list of tx types for W and H
([$([$(($ENUM:expr, $TYPE1:ident, $TYPE2:ident)),*]),*], $W:expr, $H:expr,
Expand Down Expand Up @@ -249,7 +196,7 @@ macro_rules! impl_itx_fns {
};

($TYPES64:tt, $DIMS64:tt, $TYPES32:tt, $DIMS32:tt, $TYPES16:tt, $DIMS16:tt,
$TYPES84:tt, $DIMS84:tt, $OPT:tt) => {
$TYPES84:tt, $DIMS84:tt, $TYPES4:tt, $DIMS4:tt, $OPT:tt) => {
// Make 2d list of tx types for each set of dimensions. Each set of
// dimensions uses a superset of the previous set of tx types.
impl_itx_fns!([$TYPES64], $DIMS64, $OPT);
Expand All @@ -258,15 +205,53 @@ macro_rules! impl_itx_fns {
impl_itx_fns!(
[$TYPES64, $TYPES32, $TYPES16, $TYPES84], $DIMS84, $OPT
);
impl_itx_fns!(
[$TYPES64, $TYPES32, $TYPES16, $TYPES84, $TYPES4], $DIMS4, $OPT
);

// Pool all of the dimensions together to create a table for each cpu
// feature level.
create_wxh_tables!(
[$DIMS64, $DIMS32, $DIMS16, $DIMS84], $OPT
[$DIMS64, $DIMS32, $DIMS16, $DIMS84, $DIMS4], $OPT
);
};
}

impl_itx_fns!(
// 64x
[(TxType::DCT_DCT, dct, dct)],
[(64, 64), (64, 32), (32, 64), (16, 64), (64, 16)],
// 32x
[(TxType::IDTX, identity, identity)],
[(32, 32), (32, 16), (16, 32), (32, 8), (8, 32)],
// 16x16
[
(TxType::DCT_ADST, dct, adst),
(TxType::ADST_DCT, adst, dct),
(TxType::DCT_FLIPADST, dct, flipadst),
(TxType::FLIPADST_DCT, flipadst, dct),
(TxType::V_DCT, dct, identity),
(TxType::H_DCT, identity, dct),
(TxType::ADST_ADST, adst, adst),
(TxType::ADST_FLIPADST, adst, flipadst),
(TxType::FLIPADST_ADST, flipadst, adst),
(TxType::FLIPADST_FLIPADST, flipadst, flipadst)
],
[(16, 16)],
// 8x, 4x and 16x (minus 16x16 and 4x4)
[
(TxType::V_ADST, adst, identity),
(TxType::H_ADST, identity, adst),
(TxType::V_FLIPADST, flipadst, identity),
(TxType::H_FLIPADST, identity, flipadst)
],
[(16, 8), (8, 16), (16, 4), (4, 16), (8, 8), (8, 4), (4, 8)],
// 4x4
[(TxType::WHT_WHT, wht, wht)],
[(4, 4)],
[(avx2, AVX2)]
);

impl_itx_fns!(
// 64x
[(TxType::DCT_DCT, dct, dct)],
Expand Down Expand Up @@ -296,13 +281,35 @@ impl_itx_fns!(
(TxType::H_FLIPADST, identity, flipadst)
],
[(16, 8), (8, 16), (16, 4), (4, 16), (8, 8), (8, 4), (4, 8), (4, 4)],
[(avx512icl, AVX512ICL), (avx2, AVX2), (ssse3, SSSE3)]
// 4x4
[],
[],
[(avx512icl, AVX512ICL), (ssse3, SSSE3)]
);

impl_itx_fns!(
// 64x
[],
[],
// 32x
[],
[],
// 16x16
[],
[],
// 8x, 4x and 16x (minus 16x16 and 4x4)
[],
[],
// 4x4
[(TxType::WHT_WHT, wht, wht)],
[(4, 4)],
[(sse2, SSE2)]
);

cpu_function_lookup_table!(
INV_TXFM_FNS: [[[Option<InvTxfmFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL]],
default: [[None; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL],
[SSSE3, AVX2, AVX512ICL]
[SSE2, SSSE3, AVX2, AVX512ICL]
);

macro_rules! impl_itx_hbd_fns {
Expand All @@ -321,7 +328,7 @@ macro_rules! impl_itx_hbd_fns {
};

($TYPES64:tt, $DIMS64:tt, $TYPES32:tt, $DIMS32:tt, $TYPES16:tt, $DIMS16:tt,
$TYPES84:tt, $DIMS84:tt, $EXT:ident, $OPT:tt) => {
$TYPES84:tt, $DIMS84:tt, $TYPES4:tt, $DIMS4:tt, $EXT:ident, $OPT:tt) => {
// Make 2d list of tx types for each set of dimensions. Each set of
// dimensions uses a superset of the previous set of tx types.
impl_itx_hbd_fns!([$TYPES64], $DIMS64, $OPT);
Expand All @@ -330,11 +337,14 @@ macro_rules! impl_itx_hbd_fns {
impl_itx_hbd_fns!(
[$TYPES64, $TYPES32, $TYPES16, $TYPES84], $DIMS84, $OPT
);
impl_itx_hbd_fns!(
[$TYPES64, $TYPES32, $TYPES16, $TYPES84, $TYPES4], $DIMS4, $OPT
);

// Pool all of the dimensions together to create a table for each cpu
// feature level.
create_wxh_hbd_tables!(
[$DIMS64, $DIMS32, $DIMS16, $DIMS84], $EXT, $OPT
[$DIMS64, $DIMS32, $DIMS16, $DIMS84, $DIMS4], $EXT, $OPT
);
};
}
Expand Down Expand Up @@ -368,6 +378,9 @@ impl_itx_hbd_fns!(
(TxType::H_FLIPADST, identity, flipadst)
],
[(16, 8), (8, 16), (8, 8)],
// 4x4
[],
[],
_10,
[(10, avx512icl, AVX512ICL)]
);
Expand Down Expand Up @@ -401,7 +414,10 @@ impl_itx_hbd_fns!(
(TxType::H_FLIPADST, identity, flipadst)
],
[(16, 8), (8, 16), (16, 4), (4, 16), (8, 8), (8, 4), (4, 8), (4, 4)],
_10,
// 4x4
[],
[],
_10_,
[(10, avx2, AVX2)]
);

Expand Down Expand Up @@ -434,14 +450,71 @@ impl_itx_hbd_fns!(
(TxType::H_FLIPADST, identity, flipadst)
],
[(16, 8), (8, 16), (16, 4), (4, 16), (8, 8), (8, 4), (4, 8), (4, 4)],
// 4x4
[],
[],
_10,
[(16, sse4, SSE4_1)]
);

impl_itx_hbd_fns!(
// 64x
[],
[],
// 32x
[],
[],
// 16x16
[],
[],
// 8x, 4x and 16x (minus 16x16 and 4x4)
[],
[],
// 4x4
[(TxType::WHT_WHT, wht, wht)],
[(4, 4)],
_16,
[(16, sse2, SSE2), (16, avx2, AVX2)]
);

const INV_TXFM_HBD_FNS_10_SSE2: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL];
TxSize::TX_SIZES_ALL] = INV_TXFM_HBD_FNS_16_SSE2;
const INV_TXFM_HBD_FNS_12_SSE2: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL];
TxSize::TX_SIZES_ALL] = INV_TXFM_HBD_FNS_16_SSE2;

const fn merge_hbd_fns(
a: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL],
b: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL],
) -> [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL] {
let mut out = b;
let mut tx_size = 0;
loop {
let mut tx_type = 0;
loop {
if a[tx_size][tx_type].is_some() {
out[tx_size][tx_type] = a[tx_size][tx_type];
}
tx_type += 1;
if tx_type == TX_TYPES_PLUS_LL {
break;
}
}
tx_size += 1;
if tx_size == TxSize::TX_SIZES_ALL {
break;
}
}
out
}

const INV_TXFM_HBD_FNS_10_AVX2: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL];
TxSize::TX_SIZES_ALL] =
merge_hbd_fns(INV_TXFM_HBD_FNS_10__AVX2, INV_TXFM_HBD_FNS_16_AVX2);

cpu_function_lookup_table!(
INV_TXFM_HBD_FNS_10: [[[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL]],
default: [[None; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL],
[SSE4_1, AVX2, AVX512ICL]
[SSE2, SSE4_1, AVX2, AVX512ICL]
);

impl_itx_hbd_fns!(
Expand Down Expand Up @@ -472,12 +545,19 @@ impl_itx_hbd_fns!(
(TxType::H_FLIPADST, identity, flipadst)
],
[(16, 8), (8, 16), (16, 4), (4, 16), (8, 8), (8, 4), (4, 8), (4, 4)],
_12,
// 4x4
[],
[],
_12_,
[(12, avx2, AVX2)]
);

const INV_TXFM_HBD_FNS_12_AVX2: [[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL];
TxSize::TX_SIZES_ALL] =
merge_hbd_fns(INV_TXFM_HBD_FNS_12__AVX2, INV_TXFM_HBD_FNS_16_AVX2);

cpu_function_lookup_table!(
INV_TXFM_HBD_FNS_12: [[[Option<InvTxfmHBDFunc>; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL]],
default: [[None; TX_TYPES_PLUS_LL]; TxSize::TX_SIZES_ALL],
[AVX2]
[SSE2, AVX2]
);

0 comments on commit 7738eb0

Please sign in to comment.