commit 323d535d6a45e874d5d642dd6442e61e3f13b75e
author Sarah Parker <sarahparker@google.com> Thu Aug 17 14:58:03 2017 -0700
committer Fred BARBIER <frederic.barbier@allegrodvt.com> Wed Aug 23 15:18:55 2017 +0000
tree c9dbf0059bdd2f0621a7dc4742f9b7eb621ffeca
parent 1cc50bc2ffd1bfff4a2b68a5f8d63c55fe6e0a21

Prevent illegal signaling of gm types above AFFINE

BUG=aomedia:693

Change-Id: I09b34abc41ee6f85619f5a05f668e06491e542f0

av1/common/mv.h

diff --git a/av1/common/mv.h b/av1/common/mv.h
index dabfc0e..ab35880 100644
--- a/av1/common/mv.h
+++ b/av1/common/mv.h
@@ -88,10 +88,12 @@
 // GLOBAL_TRANS_TYPES 7 - up to full homography
 #define GLOBAL_TRANS_TYPES 4
 
+#if GLOBAL_TRANS_TYPES > 4
 // First bit indicates whether using identity or not
 // GLOBAL_TYPE_BITS=ceiling(log2(GLOBAL_TRANS_TYPES-1)) is the
 // number of bits needed to cover the remaining possibilities
 #define GLOBAL_TYPE_BITS (get_msb(2 * GLOBAL_TRANS_TYPES - 3))
+#endif  // GLOBAL_TRANS_TYPES > 4
 
 typedef struct {
 #if CONFIG_GLOBAL_MOTION

av1/decoder/decodeframe.c

diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c
index 5950f19..dc6c959 100644
--- a/av1/decoder/decodeframe.c
+++ b/av1/decoder/decodeframe.c
@@ -4750,7 +4750,17 @@
                                       WarpedMotionParams *ref_params,
                                       aom_reader *r, int allow_hp) {
   TransformationType type = aom_read_bit(r, ACCT_STR);
-  if (type != IDENTITY) type += aom_read_literal(r, GLOBAL_TYPE_BITS, ACCT_STR);
+  if (type != IDENTITY) {
+#if GLOBAL_TRANS_TYPES > 4
+    type += aom_read_literal(r, GLOBAL_TYPE_BITS, ACCT_STR);
+#else
+    if (aom_read_bit(r, ACCT_STR))
+      type = ROTZOOM;
+    else
+      aom_read_bit(r, ACCT_STR) ? TRANSLATION : AFFINE;
+#endif  // GLOBAL_TRANS_TYPES > 4
+  }
+
   int trans_bits;
   int trans_dec_factor;
   int trans_prec_diff;

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 0c211b1..4c080c7 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -4535,7 +4535,14 @@
   int trans_bits;
   int trans_prec_diff;
   aom_write_bit(w, type != IDENTITY);
-  if (type != IDENTITY) aom_write_literal(w, type - 1, GLOBAL_TYPE_BITS);
+  if (type != IDENTITY) {
+#if GLOBAL_TRANS_TYPES > 4
+    aom_write_literal(w, type - 1, GLOBAL_TYPE_BITS);
+#else
+    aom_write_bit(w, type == ROTZOOM);
+    if (type != ROTZOOM) aom_write_bit(w, type == TRANSLATION);
+#endif  // GLOBAL_TRANS_TYPES > 4
+  }
 
   switch (type) {
     case HOMOGRAPHY:

av1/encoder/rd.c

diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index 419a47f..8a2a63c 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -569,8 +569,17 @@
 #if CONFIG_GLOBAL_MOTION
   if (cpi->oxcf.pass != 1) {
     for (int i = 0; i < TRANS_TYPES; ++i)
+#if GLOBAL_TRANS_TYPES > 4
       cpi->gmtype_cost[i] = (1 + (i > 0 ? GLOBAL_TYPE_BITS : 0))
                             << AV1_PROB_COST_SHIFT;
+#else
+      // IDENTITY: 1 bit
+      // TRANSLATION: 3 bits
+      // ROTZOOM: 2 bits
+      // AFFINE: 3 bits
+      cpi->gmtype_cost[i] = (1 + (i > 0 ? (i == ROTZOOM ? 1 : 2) : 0))
+                            << AV1_PROB_COST_SHIFT;
+#endif  // GLOBAL_TRANS_TYPES > 4
   }
 #endif  // CONFIG_GLOBAL_MOTION
 }