ART: Additional verifier flags for invoke-custom

Updates verifier flags for invoke-custom instructions and adds
static_asserts to enforce at build time.

Bug: 37766116
Test: m
Change-Id: I16f1411aabaa4baa7fdeb00411912487aa2f07af
diff --git a/runtime/dex_instruction.cc b/runtime/dex_instruction.cc
index 091085a..9f34c12 100644
--- a/runtime/dex_instruction.cc
+++ b/runtime/dex_instruction.cc
@@ -515,6 +515,30 @@
   return os.str();
 }
 
+// Add some checks that ensure the flags make sense. We need a subclass to be in the context of
+// Instruction. Otherwise the flags from the instruction list don't work.
+struct InstructionStaticAsserts : private Instruction {
+  #define IMPLIES(a, b) (!(a) || (b))
+
+  #define VAR_ARGS_CHECK(o, c, pname, f, i, a, v) \
+    static_assert(IMPLIES((f) == k35c || (f) == k45cc, \
+                          ((v) & (kVerifyVarArg | kVerifyVarArgNonZero)) != 0), \
+                  "Missing var-arg verification");
+  #include "dex_instruction_list.h"
+    DEX_INSTRUCTION_LIST(VAR_ARGS_CHECK)
+  #undef DEX_INSTRUCTION_LIST
+  #undef VAR_ARGS_CHECK
+
+  #define VAR_ARGS_RANGE_CHECK(o, c, pname, f, i, a, v) \
+    static_assert(IMPLIES((f) == k3rc || (f) == k4rcc, \
+                          ((v) & (kVerifyVarArgRange | kVerifyVarArgRangeNonZero)) != 0), \
+                  "Missing var-arg verification");
+  #include "dex_instruction_list.h"
+    DEX_INSTRUCTION_LIST(VAR_ARGS_RANGE_CHECK)
+  #undef DEX_INSTRUCTION_LIST
+  #undef VAR_ARGS_RANGE_CHECK
+};
+
 std::ostream& operator<<(std::ostream& os, const Instruction::Code& code) {
   return os << Instruction::Name(code);
 }
diff --git a/runtime/dex_instruction_list.h b/runtime/dex_instruction_list.h
index a5ce3c2..11dc7e2 100644
--- a/runtime/dex_instruction_list.h
+++ b/runtime/dex_instruction_list.h
@@ -271,8 +271,8 @@
   V(0xF9, UNUSED_F9, "unused-f9", k10x, kIndexUnknown, 0, kVerifyError) \
   V(0xFA, INVOKE_POLYMORPHIC, "invoke-polymorphic", k45cc, kIndexMethodAndProtoRef, kContinue | kThrow | kInvoke, kVerifyRegBMethod | kVerifyVarArgNonZero | kVerifyRegHPrototype) \
   V(0xFB, INVOKE_POLYMORPHIC_RANGE, "invoke-polymorphic/range", k4rcc, kIndexMethodAndProtoRef, kContinue | kThrow | kInvoke, kVerifyRegBMethod | kVerifyVarArgRangeNonZero | kVerifyRegHPrototype) \
-  V(0xFC, INVOKE_CUSTOM, "invoke-custom", k35c, kIndexCallSiteRef, kContinue | kThrow, kVerifyRegBCallSite) \
-  V(0xFD, INVOKE_CUSTOM_RANGE, "invoke-custom/range", k3rc, kIndexCallSiteRef, kContinue | kThrow, kVerifyRegBCallSite) \
+  V(0xFC, INVOKE_CUSTOM, "invoke-custom", k35c, kIndexCallSiteRef, kContinue | kThrow, kVerifyRegBCallSite | kVerifyVarArg) \
+  V(0xFD, INVOKE_CUSTOM_RANGE, "invoke-custom/range", k3rc, kIndexCallSiteRef, kContinue | kThrow, kVerifyRegBCallSite | kVerifyVarArgRange) \
   V(0xFE, UNUSED_FE, "unused-fe", k10x, kIndexUnknown, 0, kVerifyError) \
   V(0xFF, UNUSED_FF, "unused-ff", k10x, kIndexUnknown, 0, kVerifyError)