ART: Add IRT table size limit and overflow checks

Ensure that we don't waste too much space on tables (currently allow
128MB). Also be defensive and check for overflow when creating or
resizing.

Bug: 62223672
Test: m test-art-host
Change-Id: I60468a79d7d9dcb54767900323c2c50e79df35f3
diff --git a/runtime/indirect_reference_table.cc b/runtime/indirect_reference_table.cc
index c852d5a..cd8f84a 100644
--- a/runtime/indirect_reference_table.cc
+++ b/runtime/indirect_reference_table.cc
@@ -34,6 +34,9 @@
 static constexpr bool kDumpStackOnNonLocalReference = false;
 static constexpr bool kDebugIRT = false;
 
+// Maximum table size we allow.
+static constexpr size_t kMaxTableSizeInBytes = 128 * MB;
+
 const char* GetIndirectRefKindString(const IndirectRefKind& kind) {
   switch (kind) {
     case kHandleScopeOrInvalid:
@@ -71,6 +74,9 @@
   CHECK(error_msg != nullptr);
   CHECK_NE(desired_kind, kHandleScopeOrInvalid);
 
+  // Overflow and maximum check.
+  CHECK_LE(max_count, kMaxTableSizeInBytes / sizeof(IrtEntry));
+
   const size_t table_bytes = max_count * sizeof(IrtEntry);
   table_mem_map_.reset(MemMap::MapAnonymous("indirect ref table", nullptr, table_bytes,
                                             PROT_READ | PROT_WRITE, false, false, error_msg));
@@ -203,6 +209,13 @@
 bool IndirectReferenceTable::Resize(size_t new_size, std::string* error_msg) {
   CHECK_GT(new_size, max_entries_);
 
+  constexpr size_t kMaxEntries = kMaxTableSizeInBytes / sizeof(IrtEntry);
+  if (new_size > kMaxEntries) {
+    *error_msg = android::base::StringPrintf("Requested size exceeds maximum: %zu", new_size);
+    return false;
+  }
+  // Note: the above check also ensures that there is no overflow below.
+
   const size_t table_bytes = new_size * sizeof(IrtEntry);
   std::unique_ptr<MemMap> new_map(MemMap::MapAnonymous("indirect ref table",
                                                        nullptr,
@@ -247,6 +260,14 @@
     }
 
     // Try to double space.
+    if (std::numeric_limits<size_t>::max() / 2 < max_entries_) {
+      LOG(FATAL) << "JNI ERROR (app bug): " << kind_ << " table overflow "
+                 << "(max=" << max_entries_ << ")" << std::endl
+                 << MutatorLockedDumpable<IndirectReferenceTable>(*this)
+                << " Resizing failed: exceeds size_t";
+      UNREACHABLE();
+    }
+
     std::string error_msg;
     if (!Resize(max_entries_ * 2, &error_msg)) {
       LOG(FATAL) << "JNI ERROR (app bug): " << kind_ << " table overflow "
diff --git a/runtime/jni_internal_test.cc b/runtime/jni_internal_test.cc
index 08d1eeb..3e9907d 100644
--- a/runtime/jni_internal_test.cc
+++ b/runtime/jni_internal_test.cc
@@ -1962,6 +1962,20 @@
   check_jni_abort_catcher.Check("use of deleted local reference");
 }
 
+TEST_F(JniInternalTest, PushLocalFrame_LimitAndOverflow) {
+  // Try a very large value that should fail.
+  ASSERT_NE(JNI_OK, env_->PushLocalFrame(std::numeric_limits<jint>::max()));
+
+  // On 32-bit, also check for some overflow conditions.
+#ifndef __LP64__
+  ASSERT_EQ(JNI_OK, env_->PushLocalFrame(10));
+  ASSERT_NE(JNI_OK, env_->PushLocalFrame(std::numeric_limits<jint>::max() - 10));
+  ASSERT_TRUE(env_->ExceptionCheck());
+  env_->ExceptionClear();
+  EXPECT_EQ(env_->PopLocalFrame(nullptr), nullptr);
+#endif
+}
+
 TEST_F(JniInternalTest, NewGlobalRef_nullptr) {
   EXPECT_EQ(env_->NewGlobalRef(nullptr), nullptr);
 }