8247252: TestCombinedCompressedFlags.java failed src/hotspot/share/services/virtualMemoryTracker.cpp:388 Error: ShouldNotReachHere()

Reviewed-by: minqi, dholmes
diff --git a/src/hotspot/share/memory/metaspaceShared.cpp b/src/hotspot/share/memory/metaspaceShared.cpp
index 6b8d156..b2e98be 100644
--- a/src/hotspot/share/memory/metaspaceShared.cpp
+++ b/src/hotspot/share/memory/metaspaceShared.cpp
@@ -2270,6 +2270,7 @@
                                                                  class_space_rs);
   if (mapped_base_address == NULL) {
     result = MAP_ARCHIVE_MMAP_FAILURE;
+    log_debug(cds)("Failed to reserve spaces (use_requested_addr=%u)", (unsigned)use_requested_addr);
   } else {
 
 #ifdef ASSERT
@@ -2473,6 +2474,8 @@
     if (archive_space_rs.is_reserved()) {
       assert(base_address == NULL ||
              (address)archive_space_rs.base() == base_address, "Sanity");
+      // Register archive space with NMT.
+      MemTracker::record_virtual_memory_type(archive_space_rs.base(), mtClassShared);
       return archive_space_rs.base();
     }
     return NULL;
diff --git a/src/hotspot/share/services/virtualMemoryTracker.cpp b/src/hotspot/share/services/virtualMemoryTracker.cpp
index afba706..76cbf27 100644
--- a/src/hotspot/share/services/virtualMemoryTracker.cpp
+++ b/src/hotspot/share/services/virtualMemoryTracker.cpp
@@ -30,6 +30,7 @@
 #include "services/memTracker.hpp"
 #include "services/threadStackTracker.hpp"
 #include "services/virtualMemoryTracker.hpp"
+#include "utilities/ostream.hpp"
 
 size_t VirtualMemorySummary::_snapshot[CALC_OBJ_SIZE_IN_TYPE(VirtualMemorySnapshot, size_t)];
 
@@ -288,7 +289,9 @@
 }
 
 void ReservedMemoryRegion::set_flag(MEMFLAGS f) {
-  assert((flag() == mtNone || flag() == f), "Overwrite memory type");
+  assert((flag() == mtNone || flag() == f),
+         "Overwrite memory type for region [" PTR_FORMAT "-" PTR_FORMAT "), %u->%u.",
+         p2i(base()), p2i(end()), (unsigned)flag(), (unsigned)f);
   if (flag() != f) {
     VirtualMemorySummary::move_reserved_memory(flag(), f, size());
     VirtualMemorySummary::move_committed_memory(flag(), f, committed_size());
@@ -385,6 +388,13 @@
         return true;
       }
 
+      // Print some more details. Don't use UL here to avoid circularities.
+#ifdef ASSERT
+      tty->print_cr("Error: existing region: [" PTR_FORMAT "-" PTR_FORMAT "), flag %u.\n"
+                    "       new region: [" PTR_FORMAT "-" PTR_FORMAT "), flag %u.",
+                    p2i(reserved_rgn->base()), p2i(reserved_rgn->end()), (unsigned)reserved_rgn->flag(),
+                    p2i(base_addr), p2i(base_addr + size), (unsigned)flag);
+#endif
       ShouldNotReachHere();
       return false;
     }
diff --git a/test/hotspot/jtreg/runtime/cds/SharedBaseAddress.java b/test/hotspot/jtreg/runtime/cds/SharedBaseAddress.java
index 12b55db..ad3fb65 100644
--- a/test/hotspot/jtreg/runtime/cds/SharedBaseAddress.java
+++ b/test/hotspot/jtreg/runtime/cds/SharedBaseAddress.java
@@ -58,7 +58,8 @@
                 .setArchiveName(filename)
                 .addPrefix("-XX:SharedBaseAddress=" + testEntry)
                 .addPrefix("-Xlog:cds=debug")
-                .addPrefix("-Xlog:cds+reloc=debug");
+                .addPrefix("-Xlog:cds+reloc=debug")
+                .addPrefix("-XX:NativeMemoryTracking=detail");
 
             CDSTestUtils.createArchiveAndCheck(opts);
             OutputAnalyzer out = CDSTestUtils.runWithArchiveAndCheck(opts);
diff --git a/test/hotspot/jtreg/runtime/cds/appcds/ArchiveRelocationTest.java b/test/hotspot/jtreg/runtime/cds/appcds/ArchiveRelocationTest.java
index c43e383..59d642d 100644
--- a/test/hotspot/jtreg/runtime/cds/appcds/ArchiveRelocationTest.java
+++ b/test/hotspot/jtreg/runtime/cds/appcds/ArchiveRelocationTest.java
@@ -68,10 +68,11 @@
         String runRelocArg  = run_reloc  ? forceRelocation : "-showversion";
         String logArg = "-Xlog:cds=debug,cds+reloc=debug";
         String unlockArg = "-XX:+UnlockDiagnosticVMOptions";
+        String nmtArg = "-XX:NativeMemoryTracking=detail";
 
         OutputAnalyzer out = TestCommon.dump(appJar,
                                              TestCommon.list(mainClass),
-                                             unlockArg, dumpRelocArg, logArg);
+                                             unlockArg, dumpRelocArg, logArg, nmtArg);
         if (dump_reloc) {
             out.shouldContain("ArchiveRelocationMode == 1: always allocate class space at an alternative address");
             out.shouldContain("Relocating archive from");
diff --git a/test/hotspot/jtreg/runtime/cds/appcds/TestCombinedCompressedFlags.java b/test/hotspot/jtreg/runtime/cds/appcds/TestCombinedCompressedFlags.java
index 4ae8b0d..4f3681d 100644
--- a/test/hotspot/jtreg/runtime/cds/appcds/TestCombinedCompressedFlags.java
+++ b/test/hotspot/jtreg/runtime/cds/appcds/TestCombinedCompressedFlags.java
@@ -163,7 +163,8 @@
                       new String[] {"Hello"},
                       getCompressedOopsArg(t.dumpArg.useCompressedOops),
                       getCompressedClassPointersArg(t.dumpArg.useCompressedClassPointers),
-                      "-Xlog:cds");
+                      "-Xlog:cds",
+                      "-XX:NativeMemoryTracking=detail");
             out.shouldContain("Dumping shared data to file:");
             out.shouldHaveExitValue(0);
 
@@ -172,6 +173,7 @@
                                       "-cp",
                                       helloJar,
                                       "-Xlog:cds",
+                                      "-XX:NativeMemoryTracking=detail",
                                       getCompressedOopsArg(c.useCompressedOops),
                                       getCompressedClassPointersArg(c.useCompressedClassPointers),
                                       "Hello");