8242271: Shenandoah: add test to verify GC mode unlock

Reviewed-by: rkennke
diff --git a/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java b/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java
new file mode 100644
index 0000000..c338b27
--- /dev/null
+++ b/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2020, Red Hat, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test TestModeUnlock
+ * @summary Test that Shenandoah modes are unlocked properly
+ * @key gc
+ * @requires vm.gc.Shenandoah & !vm.graal.enabled
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ *          java.management
+ * @run driver TestModeUnlock
+ */
+
+import jdk.test.lib.process.ProcessTools;
+import jdk.test.lib.process.OutputAnalyzer;
+
+public class TestModeUnlock {
+
+    enum Mode {
+        PRODUCT,
+        DIAGNOSTIC,
+        EXPERIMENTAL,
+    }
+
+    public static void main(String[] args) throws Exception {
+        testWith("-XX:ShenandoahGCMode=normal",  Mode.PRODUCT);
+        testWith("-XX:ShenandoahGCMode=iu",      Mode.EXPERIMENTAL);
+        testWith("-XX:ShenandoahGCMode=passive", Mode.DIAGNOSTIC);
+    }
+
+    private static void testWith(String h, Mode mode) throws Exception {
+        if (false) { // When ShenandoahGC is experimental flag, this makes no sense to test
+            ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(
+                    "-XX:-UnlockDiagnosticVMOptions",
+                    "-XX:-UnlockExperimentalVMOptions",
+                    "-XX:+UseShenandoahGC",
+                    h,
+                    "-version"
+            );
+            OutputAnalyzer output = new OutputAnalyzer(pb.start());
+            switch (mode) {
+                case PRODUCT:
+                    output.shouldHaveExitValue(0);
+                    break;
+                case DIAGNOSTIC:
+                case EXPERIMENTAL:
+                    output.shouldNotHaveExitValue(0);
+                    break;
+            }
+        }
+
+        if (false) { // When ShenandoahGC is experimental flag, this makes no sense to test
+            ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(
+                    "-XX:+UnlockDiagnosticVMOptions",
+                    "-XX:-UnlockExperimentalVMOptions",
+                    "-XX:+UseShenandoahGC",
+                    h,
+                    "-version"
+            );
+            OutputAnalyzer output = new OutputAnalyzer(pb.start());
+            switch (mode) {
+                case PRODUCT:
+                case DIAGNOSTIC:
+                    output.shouldHaveExitValue(0);
+                    break;
+                case EXPERIMENTAL:
+                    output.shouldNotHaveExitValue(0);
+                    break;
+            }
+        }
+
+        {
+            ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(
+                    "-XX:-UnlockDiagnosticVMOptions",
+                    "-XX:+UnlockExperimentalVMOptions",
+                    "-XX:+UseShenandoahGC",
+                    h,
+                    "-version"
+            );
+            OutputAnalyzer output = new OutputAnalyzer(pb.start());
+            switch (mode) {
+                case PRODUCT:
+                case EXPERIMENTAL:
+                    output.shouldHaveExitValue(0);
+                    break;
+                case DIAGNOSTIC:
+                    output.shouldNotHaveExitValue(0);
+                    break;
+            }
+        }
+    }
+
+}