Added code to handle cases with qp less than 10
When qp goes less than 10, all I16x16 mode evaluations
are disabled and I4x4 evaluations are enabled irrespective
of preset. This will ensure that the residual will not
exceed the supported range of entropy
Minimum Qp cannot be set less than 4 now. When QP goes lesser,
the residual may exceed the supported range even with I4x4 enabled.
Change-Id: I25b404fcd9c9e9dbdd77679280968635ee047eb3
diff --git a/encoder/ih264e_api.c b/encoder/ih264e_api.c
index a8de1ec..1cb2ff6 100644
--- a/encoder/ih264e_api.c
+++ b/encoder/ih264e_api.c
@@ -1561,7 +1561,11 @@
return IV_FAIL;
}
- if ((ps_ip->s_ive_ip.u4_i_qp_min > ps_ip->s_ive_ip.u4_i_qp_max)
+ /* We donot support QP < 4 */
+ if ((ps_ip->s_ive_ip.u4_i_qp_min < 4)
+ || (ps_ip->s_ive_ip.u4_p_qp_min < 4)
+ || (ps_ip->s_ive_ip.u4_b_qp_min < 4)
+ || (ps_ip->s_ive_ip.u4_i_qp_min > ps_ip->s_ive_ip.u4_i_qp_max)
|| (ps_ip->s_ive_ip.u4_p_qp_min > ps_ip->s_ive_ip.u4_p_qp_max)
|| (ps_ip->s_ive_ip.u4_b_qp_min > ps_ip->s_ive_ip.u4_b_qp_max))
{
diff --git a/encoder/ih264e_process.c b/encoder/ih264e_process.c
index ff6846a..850cefc 100644
--- a/encoder/ih264e_process.c
+++ b/encoder/ih264e_process.c
@@ -1916,42 +1916,63 @@
/* temp variables */
WORD32 ctxt_sel = ps_proc->i4_encode_api_call_cnt % MAX_CTXT_SETS;
- /* list of modes for evaluation */
+ /*
+ * list of modes for evaluation
+ * -------------------------------------------------------------------------
+ * Note on enabling I4x4 and I16x16
+ * At very low QP's the hadamard transform in I16x16 will push up the maximum
+ * coeff value very high. CAVLC may not be able to represent the value and
+ * hence the stream may not be decodable in some clips.
+ * Hence at low QPs, we will enable I4x4 and disable I16x16 irrespective of preset.
+ */
if (ps_proc->i4_slice_type == ISLICE)
{
- /* enable intra 16x16 */
- u4_valid_modes |= ps_codec->s_cfg.u4_enable_intra_16x16 ? (1 << I16x16) : 0;
+ if (ps_proc->u4_frame_qp > 10)
+ {
+ /* enable intra 16x16 */
+ u4_valid_modes |= ps_codec->s_cfg.u4_enable_intra_16x16 ? (1 << I16x16) : 0;
- /* enable intra 8x8 */
- u4_valid_modes |= ps_codec->s_cfg.u4_enable_intra_8x8 ? (1 << I8x8) : 0;
+ /* enable intra 8x8 */
+ u4_valid_modes |= ps_codec->s_cfg.u4_enable_intra_8x8 ? (1 << I8x8) : 0;
+ }
/* enable intra 4x4 */
u4_valid_modes |= ps_codec->s_cfg.u4_enable_intra_4x4 ? (1 << I4x4) : 0;
+ u4_valid_modes |= (ps_proc->u4_frame_qp <= 10) << I4x4;
+
}
else if (ps_proc->i4_slice_type == PSLICE)
{
- /* enable intra 16x16 */
- u4_valid_modes |= ps_codec->s_cfg.u4_enable_intra_16x16 ? (1 << I16x16) : 0;
+ if (ps_proc->u4_frame_qp > 10)
+ {
+ /* enable intra 16x16 */
+ u4_valid_modes |= ps_codec->s_cfg.u4_enable_intra_16x16 ? (1 << I16x16) : 0;
+ }
/* enable intra 4x4 */
if (ps_codec->s_cfg.u4_enc_speed_preset == IVE_SLOWEST)
{
u4_valid_modes |= ps_codec->s_cfg.u4_enable_intra_4x4 ? (1 << I4x4) : 0;
}
+ u4_valid_modes |= (ps_proc->u4_frame_qp <= 10) << I4x4;
/* enable inter P16x16 */
u4_valid_modes |= (1 << P16x16);
}
else if (ps_proc->i4_slice_type == BSLICE)
{
- /* enable intra 16x16 */
- u4_valid_modes |= ps_codec->s_cfg.u4_enable_intra_16x16 ? (1 << I16x16) : 0;
+ if (ps_proc->u4_frame_qp > 10)
+ {
+ /* enable intra 16x16 */
+ u4_valid_modes |= ps_codec->s_cfg.u4_enable_intra_16x16 ? (1 << I16x16) : 0;
+ }
/* enable intra 4x4 */
if (ps_codec->s_cfg.u4_enc_speed_preset == IVE_SLOWEST)
{
u4_valid_modes |= ps_codec->s_cfg.u4_enable_intra_4x4 ? (1 << I4x4) : 0;
}
+ u4_valid_modes |= (ps_proc->u4_frame_qp <= 10) << I4x4;
/* enable inter B16x16 */
u4_valid_modes |= (1 << B16x16);
diff --git a/test/encoder/app.h b/test/encoder/app.h
index 01edd8b..ad45f5a 100644
--- a/test/encoder/app.h
+++ b/test/encoder/app.h
@@ -73,7 +73,7 @@
#define DEFAULT_MAX_REF_FRM 2
#define DEFAULT_MAX_REORDER_FRM 0
-#define DEFAULT_QP_MIN 0
+#define DEFAULT_QP_MIN 4
#define DEFAULT_QP_MAX 51
#define DEFAULT_MAX_BITRATE 20000000
#define DEFAULT_NUM_BFRAMES 0
