8022968: Some codecache allocation failures don't result in invoking the sweeper

Add calls to CompileBroker::handle_full_code_cache() or fail gracefully whenever allocation in the code cache fails.

Reviewed-by: iveresov, vlivanov
diff --git a/hotspot/src/share/vm/code/vtableStubs.cpp b/hotspot/src/share/vm/code/vtableStubs.cpp
index ac99da4..07a1083 100644
--- a/hotspot/src/share/vm/code/vtableStubs.cpp
+++ b/hotspot/src/share/vm/code/vtableStubs.cpp
@@ -24,6 +24,7 @@
 
 #include "precompiled.hpp"
 #include "code/vtableStubs.hpp"
+#include "compiler/compileBroker.hpp"
 #include "compiler/disassembler.hpp"
 #include "memory/allocation.inline.hpp"
 #include "memory/resourceArea.hpp"
@@ -62,6 +63,7 @@
    // If changing the name, update the other file accordingly.
     BufferBlob* blob = BufferBlob::create("vtable chunks", bytes);
     if (blob == NULL) {
+      CompileBroker::handle_full_code_cache();
       return NULL;
     }
     _chunk = blob->content_begin();
diff --git a/hotspot/src/share/vm/interpreter/interpreterRuntime.cpp b/hotspot/src/share/vm/interpreter/interpreterRuntime.cpp
index 42a7218..c795b11 100644
--- a/hotspot/src/share/vm/interpreter/interpreterRuntime.cpp
+++ b/hotspot/src/share/vm/interpreter/interpreterRuntime.cpp
@@ -1093,6 +1093,7 @@
 address SignatureHandlerLibrary::set_handler_blob() {
   BufferBlob* handler_blob = BufferBlob::create("native signature handlers", blob_size);
   if (handler_blob == NULL) {
+    CompileBroker::handle_full_code_cache();
     return NULL;
   }
   address handler = handler_blob->code_begin();
diff --git a/hotspot/src/share/vm/opto/compile.cpp b/hotspot/src/share/vm/opto/compile.cpp
index 9c8ae35..bd122dd 100644
--- a/hotspot/src/share/vm/opto/compile.cpp
+++ b/hotspot/src/share/vm/opto/compile.cpp
@@ -29,6 +29,7 @@
 #include "classfile/systemDictionary.hpp"
 #include "code/exceptionHandlerTable.hpp"
 #include "code/nmethod.hpp"
+#include "compiler/compileBroker.hpp"
 #include "compiler/compileLog.hpp"
 #include "compiler/disassembler.hpp"
 #include "compiler/oopMap.hpp"
@@ -555,6 +556,7 @@
     if (scratch_buffer_blob() == NULL) {
       // Let CompilerBroker disable further compilations.
       record_failure("Not enough space for scratch buffer in CodeCache");
+      CompileBroker::handle_full_code_cache();
       return;
     }
   }
diff --git a/hotspot/src/share/vm/opto/output.cpp b/hotspot/src/share/vm/opto/output.cpp
index be29ce4..a484a10 100644
--- a/hotspot/src/share/vm/opto/output.cpp
+++ b/hotspot/src/share/vm/opto/output.cpp
@@ -1163,6 +1163,7 @@
   // Have we run out of code space?
   if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) {
     C->record_failure("CodeCache is full");
+    CompileBroker::handle_full_code_cache();
     return NULL;
   }
   // Configure the code buffer.
@@ -1487,6 +1488,7 @@
       cb->insts()->maybe_expand_to_ensure_remaining(MAX_inst_size);
       if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) {
         C->record_failure("CodeCache is full");
+        CompileBroker::handle_full_code_cache();
         return;
       }
 
@@ -1643,6 +1645,7 @@
   // One last check for failed CodeBuffer::expand:
   if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) {
     C->record_failure("CodeCache is full");
+    CompileBroker::handle_full_code_cache();
     return;
   }
 
diff --git a/hotspot/src/share/vm/runtime/icache.cpp b/hotspot/src/share/vm/runtime/icache.cpp
index 02ac795..02f36d5 100644
--- a/hotspot/src/share/vm/runtime/icache.cpp
+++ b/hotspot/src/share/vm/runtime/icache.cpp
@@ -34,6 +34,9 @@
   ResourceMark rm;
 
   BufferBlob* b = BufferBlob::create("flush_icache_stub", ICache::stub_size);
+  if (b == NULL) {
+    vm_exit_out_of_memory(ICache::stub_size, OOM_MALLOC_ERROR, "CodeCache: no space for flush_icache_stub");
+  }
   CodeBuffer c(b);
 
   ICacheStubGenerator g(&c);