Commit 40f676a743 for aom
commit 40f676a7438a78417444bae01cdd03e12247bfe7
Author: Anusuya <anusuya.k@ittiam.com>
Date: Tue Jan 20 14:57:28 2026 +0530
Improve sf 'fast_intra_tx_type_search'
In this patch, a level is introduced to the sf
'fast_intra_tx_type_search' to include more transform
types during MODE_EVAL.
The storing of CDEF UV strengths is avoided for monochrome
videos to fix assertion failure.
Encoder Instruction BD-Rate Loss(%)
cpu Count Reduction(%) avg.psnr ovr.psnr ssim vmaf vmaf_neg
4 -1.78 -0.6072 -0.6154 -0.6646 -0.5226 -0.4909
5 -1.65 -0.5585 -0.5830 -0.6359 -0.5554 -0.5383
6 -1.86 -0.7323 -0.7344 -0.7886 -0.7203 -0.6906
STATS_CHANGED for speed>=4
Change-Id: I2c7944340b9d41ffbcff20d443f8f0d5cc00ea3d
diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index 35af535405..194eb396b5 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -441,6 +441,11 @@ typedef struct {
*/
int use_default_intra_tx_type;
+ /*! Whether to limit the intra transform search type to the ones in the table
+ * av1_derived_intra_tx_used_flag[INTRA_MODES].
+ */
+ int use_derived_intra_tx_type_set;
+
/*! Probability threshold used for conditionally forcing tx type*/
int default_inter_tx_type_prob_thresh;
diff --git a/av1/encoder/pickcdef.c b/av1/encoder/pickcdef.c
index 596b4a487c..918f6fd3a6 100644
--- a/av1/encoder/pickcdef.c
+++ b/av1/encoder/pickcdef.c
@@ -975,13 +975,15 @@ void av1_cdef_search(AV1_COMP *cpi) {
if (fast) {
for (int j = 0; j < cdef_info->nb_cdef_strengths; j++) {
const int luma_strength = cdef_info->cdef_strengths[j];
- const int chroma_strength = cdef_info->cdef_uv_strengths[j];
int pri_strength, sec_strength;
STORE_CDEF_FILTER_STRENGTH(cdef_info->cdef_strengths[j], pick_method,
luma_strength);
- STORE_CDEF_FILTER_STRENGTH(cdef_info->cdef_uv_strengths[j], pick_method,
- chroma_strength);
+ if (num_planes > 1) {
+ const int chroma_strength = cdef_info->cdef_uv_strengths[j];
+ STORE_CDEF_FILTER_STRENGTH(cdef_info->cdef_uv_strengths[j], pick_method,
+ chroma_strength);
+ }
}
}
diff --git a/av1/encoder/rdopt_utils.h b/av1/encoder/rdopt_utils.h
index 5214800c83..5063d34bb7 100644
--- a/av1/encoder/rdopt_utils.h
+++ b/av1/encoder/rdopt_utils.h
@@ -558,6 +558,7 @@ static inline void set_mode_eval_params(const struct AV1_COMP *cpi,
case DEFAULT_EVAL:
txfm_params->default_inter_tx_type_prob_thresh = INT_MAX;
txfm_params->use_default_intra_tx_type = 0;
+ txfm_params->use_derived_intra_tx_type_set = 0;
txfm_params->skip_txfm_level =
winner_mode_params->skip_txfm_level[DEFAULT_EVAL];
txfm_params->predict_dc_level =
@@ -576,8 +577,10 @@ static inline void set_mode_eval_params(const struct AV1_COMP *cpi,
break;
case MODE_EVAL:
txfm_params->use_default_intra_tx_type =
- (cpi->sf.tx_sf.tx_type_search.fast_intra_tx_type_search ||
+ (cpi->sf.tx_sf.tx_type_search.fast_intra_tx_type_search == 2 ||
cpi->oxcf.txfm_cfg.use_intra_default_tx_only);
+ txfm_params->use_derived_intra_tx_type_set =
+ cpi->sf.tx_sf.tx_type_search.fast_intra_tx_type_search == 1;
txfm_params->default_inter_tx_type_prob_thresh =
cpi->sf.tx_sf.tx_type_search.fast_inter_tx_type_prob_thresh;
txfm_params->skip_txfm_level =
@@ -607,6 +610,7 @@ static inline void set_mode_eval_params(const struct AV1_COMP *cpi,
case WINNER_MODE_EVAL:
txfm_params->default_inter_tx_type_prob_thresh = INT_MAX;
txfm_params->use_default_intra_tx_type = 0;
+ txfm_params->use_derived_intra_tx_type_set = 0;
txfm_params->skip_txfm_level =
winner_mode_params->skip_txfm_level[WINNER_MODE_EVAL];
txfm_params->predict_dc_level =
diff --git a/av1/encoder/speed_features.c b/av1/encoder/speed_features.c
index e4c5e94c2d..f94e324445 100644
--- a/av1/encoder/speed_features.c
+++ b/av1/encoder/speed_features.c
@@ -501,7 +501,7 @@ static void set_allintra_speed_features_framesize_independent(
sf->tpl_sf.search_method = FAST_BIGDIA;
sf->tx_sf.tx_type_search.winner_mode_tx_type_pruning = 2;
- sf->tx_sf.tx_type_search.fast_intra_tx_type_search = 1;
+ sf->tx_sf.tx_type_search.fast_intra_tx_type_search = 2;
sf->tx_sf.tx_type_search.prune_2d_txfm_mode = TX_TYPE_PRUNE_3;
sf->tx_sf.tx_type_search.prune_tx_type_est_rd = 1;
@@ -1976,7 +1976,7 @@ static void set_rt_speed_features_framesize_independent(AV1_COMP *cpi,
sf->tx_sf.inter_tx_size_search_init_depth_sqr = 1;
sf->tx_sf.tx_type_search.prune_2d_txfm_mode = TX_TYPE_PRUNE_3;
sf->tx_sf.refine_fast_tx_search_results = 0;
- sf->tx_sf.tx_type_search.fast_intra_tx_type_search = 1;
+ sf->tx_sf.tx_type_search.fast_intra_tx_type_search = 2;
sf->tx_sf.tx_type_search.use_skip_flag_prediction = 2;
sf->tx_sf.tx_type_search.winner_mode_tx_type_pruning = 4;
diff --git a/av1/encoder/speed_features.h b/av1/encoder/speed_features.h
index cae217f9a4..54e64aeefc 100644
--- a/av1/encoder/speed_features.h
+++ b/av1/encoder/speed_features.h
@@ -256,6 +256,11 @@ enum {
typedef struct {
TX_TYPE_PRUNE_MODE prune_2d_txfm_mode;
+
+ // Limit the intra transform search type.
+ // 1 : Limit the intra transform search type to the ones in the table
+ // av1_derived_intra_tx_used_flag[INTRA_MODES].
+ // 2 : Limit the intra transform search type to the default transform.
int fast_intra_tx_type_search;
// INT_MAX: Disable fast search.
diff --git a/av1/encoder/tx_search.c b/av1/encoder/tx_search.c
index 8bf926d5de..470946c8a6 100644
--- a/av1/encoder/tx_search.c
+++ b/av1/encoder/tx_search.c
@@ -1813,6 +1813,9 @@ static inline uint16_t get_tx_mask(
} else if (fast_tx_search) {
allowed_tx_mask = 0x0c01; // V_DCT, H_DCT, DCT_DCT
allowed_tx_mask &= ext_tx_used_flag;
+ } else if (!is_inter && txfm_params->use_derived_intra_tx_type_set) {
+ allowed_tx_mask = av1_derived_intra_tx_used_flag[intra_dir];
+ allowed_tx_mask &= ext_tx_used_flag;
} else {
assert(plane == 0);
allowed_tx_mask = ext_tx_used_flag;