Modified key frame detection.
Address poor key frame detection in some content.
This patch improves on poor key frame / scene cut detection observed
with some test content. The content in question was letter boxed film
style material and also had quite low contrast. For both 1080P and 4K
multiple genuine scene cuts were being missed.
The changes alter the conditions for marking a transition as a "flash" rather
than a scene change. The new code still deals well with genuine flashes as
observed in the "crew" test clip, without falsely flagging some of the
the scene cuts in the "film" test clip.
The new film test clip also had some "flash" frames caused by a lightning
effect and in one case a flash occurred right before a scene change. This
caused a misplacement of the key frame but has been addressed by a new
clause that requires the coded error for the next frame after a candidate
key frame to be lower than the current frame.
The patch also changes the way in which neutral blocks (similar inter and
inter error) are handled in the candidate key frame decision in a way which
hopefully handles the letter boxed format better.
During wider testing some film clips still had missed key frames but this
patch does improve things. In the case of the initial test clip the encoder
correctly marks all 3 scene cuts vs 0 before the patch.
Testing with our standard (mainly short single kf) derf and NF test clips
is neutral.
Change-Id: I3b7dcfe7b2fb13fd0816ea46acc3e69c8bc581b3
diff --git a/vp9/encoder/vp9_firstpass.c b/vp9/encoder/vp9_firstpass.c
index 81fbf34..750b773 100644
--- a/vp9/encoder/vp9_firstpass.c
+++ b/vp9/encoder/vp9_firstpass.c
@@ -1825,10 +1825,12 @@
// brief break in prediction (such as a flash) but subsequent frames
// are reasonably well predicted by an earlier (pre flash) frame.
// The recovery after a flash is indicated by a high pcnt_second_ref
- // compared to pcnt_inter.
+ // useage or a second ref coded error notabley lower than the last
+ // frame coded error.
return next_frame != NULL &&
- next_frame->pcnt_second_ref > next_frame->pcnt_inter &&
- next_frame->pcnt_second_ref >= 0.5;
+ ((next_frame->sr_coded_error < next_frame->coded_error) ||
+ ((next_frame->pcnt_second_ref > next_frame->pcnt_inter) &&
+ (next_frame->pcnt_second_ref >= 0.5)));
}
// Update the motion related elements to the GF arf boost calculation.
@@ -2577,17 +2579,17 @@
&next_frame, &this_frame_mv_in_out, &mv_in_out_accumulator,
&abs_mv_in_out_accumulator, &mv_ratio_accumulator);
+ // Monitor for static sections.
+ if ((rc->frames_since_key + i - 1) > 1) {
+ zero_motion_accumulator = VPXMIN(
+ zero_motion_accumulator, get_zero_motion_factor(cpi, &next_frame));
+ }
+
// Accumulate the effect of prediction quality decay.
if (!flash_detected) {
last_loop_decay_rate = loop_decay_rate;
loop_decay_rate = get_prediction_decay_rate(cpi, &next_frame);
- // Monitor for static sections.
- if ((rc->frames_since_key + i - 1) > 1) {
- zero_motion_accumulator = VPXMIN(
- zero_motion_accumulator, get_zero_motion_factor(cpi, &next_frame));
- }
-
// Break clause to detect very still sections after motion. For example,
// a static image after a fade or other transition.
if (detect_transition_to_still(cpi, i, 5, loop_decay_rate,
@@ -2755,17 +2757,11 @@
(this_frame->coded_error > (next_frame->coded_error * ERROR_SPIKE));
}
-// Threshold for use of the lagging second reference frame. High second ref
-// usage may point to a transient event like a flash or occlusion rather than
-// a real scene cut.
-#define SECOND_REF_USEAGE_THRESH 0.1
// Minimum % intra coding observed in first pass (1.0 = 100%)
#define MIN_INTRA_LEVEL 0.25
-// Minimum ratio between the % of intra coding and inter coding in the first
-// pass after discounting neutral blocks (discounting neutral blocks in this
-// way helps catch scene cuts in clips with very flat areas or letter box
-// format clips with image padding.
-#define INTRA_VS_INTER_THRESH 2.0
+// Threshold for use of the lagging second reference frame. Scene cuts do not
+// usually have a high second ref useage.
+#define SECOND_REF_USEAGE_THRESH 0.125
// Hard threshold where the first pass chooses intra for almost all blocks.
// In such a case even if the frame is not a scene cut coding a key frame
// may be a good option.
@@ -2773,12 +2769,6 @@
// Maximum threshold for the relative ratio of intra error score vs best
// inter error score.
#define KF_II_ERR_THRESHOLD 2.5
-// In real scene cuts there is almost always a sharp change in the intra
-// or inter error score.
-#define ERR_CHANGE_THRESHOLD 0.4
-// For real scene cuts we expect an improvment in the intra inter error
-// ratio in the next frame.
-#define II_IMPROVEMENT_THRESHOLD 3.5
#define KF_II_MAX 128.0
#define II_FACTOR 12.5
// Test for very low intra complexity which could cause false key frames
@@ -2790,30 +2780,21 @@
const FIRSTPASS_STATS *next_frame) {
int is_viable_kf = 0;
double pcnt_intra = 1.0 - this_frame->pcnt_inter;
- double modified_pcnt_inter =
- this_frame->pcnt_inter - this_frame->pcnt_neutral;
// Does the frame satisfy the primary criteria of a key frame?
// See above for an explanation of the test criteria.
// If so, then examine how well it predicts subsequent frames.
- if ((this_frame->pcnt_second_ref < SECOND_REF_USEAGE_THRESH) &&
- (next_frame->pcnt_second_ref < SECOND_REF_USEAGE_THRESH) &&
+ if (!detect_flash(twopass, -1) && !detect_flash(twopass, 0) &&
+ (this_frame->pcnt_second_ref < SECOND_REF_USEAGE_THRESH) &&
((this_frame->pcnt_inter < VERY_LOW_INTER_THRESH) ||
(slide_transition(this_frame, last_frame, next_frame)) ||
- ((pcnt_intra > MIN_INTRA_LEVEL) &&
- (pcnt_intra > (INTRA_VS_INTER_THRESH * modified_pcnt_inter)) &&
+ (((this_frame->coded_error > (next_frame->coded_error * 1.1)) &&
+ (this_frame->coded_error > (last_frame->coded_error * 1.1))) &&
+ (pcnt_intra > MIN_INTRA_LEVEL) &&
+ ((pcnt_intra + this_frame->pcnt_neutral) > 0.5) &&
((this_frame->intra_error /
DOUBLE_DIVIDE_CHECK(this_frame->coded_error)) <
- KF_II_ERR_THRESHOLD) &&
- ((fabs(last_frame->coded_error - this_frame->coded_error) /
- DOUBLE_DIVIDE_CHECK(this_frame->coded_error) >
- ERR_CHANGE_THRESHOLD) ||
- (fabs(last_frame->intra_error - this_frame->intra_error) /
- DOUBLE_DIVIDE_CHECK(this_frame->intra_error) >
- ERR_CHANGE_THRESHOLD) ||
- ((next_frame->intra_error /
- DOUBLE_DIVIDE_CHECK(next_frame->coded_error)) >
- II_IMPROVEMENT_THRESHOLD))))) {
+ KF_II_ERR_THRESHOLD)))) {
int i;
const FIRSTPASS_STATS *start_pos = twopass->stats_in;
FIRSTPASS_STATS local_next_frame = *next_frame;
@@ -3254,9 +3235,9 @@
FILE *fpfile;
fpfile = fopen("arf.stt", "a");
++arf_count;
- fprintf(fpfile, "%10d %10ld %10d %10d %10ld\n", cm->current_video_frame,
- rc->frames_till_gf_update_due, rc->kf_boost, arf_count,
- rc->gfu_boost);
+ fprintf(fpfile, "%10d %10ld %10d %10d %10ld %10ld\n",
+ cm->current_video_frame, rc->frames_till_gf_update_due,
+ rc->kf_boost, arf_count, rc->gfu_boost, cm->frame_type);
fclose(fpfile);
}
