| /* |
| * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved. |
| * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| * |
| * This code is free software; you can redistribute it and/or modify it |
| * under the terms of the GNU General Public License version 2 only, as |
| * published by the Free Software Foundation. Oracle designates this |
| * particular file as subject to the "Classpath" exception as provided |
| * by Oracle in the LICENSE file that accompanied this code. |
| * |
| * This code is distributed in the hope that it will be useful, but WITHOUT |
| * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| * version 2 for more details (a copy is included in the LICENSE file that |
| * accompanied this code). |
| * |
| * You should have received a copy of the GNU General Public License version |
| * 2 along with this work; if not, write to the Free Software Foundation, |
| * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| * |
| * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| * or visit www.oracle.com if you need additional information or have any |
| * questions. |
| */ |
| |
| package jdk.nashorn.internal.test.framework; |
| |
| import static jdk.nashorn.internal.test.framework.TestConfig.TEST_FAILED_LIST_FILE; |
| import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ENABLE_STRICT_MODE; |
| import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDES_FILE; |
| import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDE_LIST; |
| import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_FRAMEWORK; |
| import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ROOTS; |
| import java.io.BufferedReader; |
| import java.io.ByteArrayOutputStream; |
| import java.io.File; |
| import java.io.FileInputStream; |
| import java.io.FileOutputStream; |
| import java.io.FileReader; |
| import java.io.FileWriter; |
| import java.io.IOException; |
| import java.io.InputStreamReader; |
| import java.io.OutputStream; |
| import java.io.PrintStream; |
| import java.io.PrintWriter; |
| import java.io.StringReader; |
| import java.nio.file.FileSystems; |
| import java.nio.file.Files; |
| import java.nio.file.StandardCopyOption; |
| import java.util.ArrayList; |
| import java.util.Collections; |
| import java.util.Comparator; |
| import java.util.List; |
| import java.util.Locale; |
| import java.util.Map; |
| import java.util.Properties; |
| import java.util.Set; |
| import java.util.TreeSet; |
| import java.util.concurrent.Callable; |
| import java.util.concurrent.CancellationException; |
| import java.util.concurrent.CountDownLatch; |
| import java.util.concurrent.ExecutionException; |
| import java.util.concurrent.ExecutorService; |
| import java.util.concurrent.Executors; |
| import java.util.concurrent.Future; |
| import java.util.concurrent.TimeUnit; |
| import java.util.regex.Matcher; |
| import java.util.regex.Pattern; |
| import jdk.nashorn.internal.test.framework.TestFinder.TestFactory; |
| |
| /** |
| * Parallel test runner runs tests in multiple threads - but avoids any dependency |
| * on third-party test framework library such as TestNG. |
| */ |
| @SuppressWarnings("javadoc") |
| public class ParallelTestRunner { |
| |
| // ParallelTestRunner-specific |
| private static final String TEST_JS_THREADS = "test.js.threads"; |
| private static final String TEST_JS_REPORT_FILE = "test.js.report.file"; |
| // test262 does a lot of eval's and the JVM hates multithreaded class definition, so lower thread count is usually faster. |
| private static final int THREADS = Integer.getInteger(TEST_JS_THREADS, Runtime.getRuntime().availableProcessors() > 4 ? 4 : 2); |
| |
| private final List<ScriptRunnable> tests = new ArrayList<>(); |
| private final Set<String> orphans = new TreeSet<>(); |
| private final ExecutorService executor = Executors.newFixedThreadPool(THREADS); |
| |
| // Ctrl-C handling |
| private final CountDownLatch finishedLatch = new CountDownLatch(1); |
| private final Thread shutdownHook = new Thread() { |
| @Override |
| public void run() { |
| if (!executor.isTerminated()) { |
| executor.shutdownNow(); |
| try { |
| executor.awaitTermination(25, TimeUnit.SECONDS); |
| finishedLatch.await(5, TimeUnit.SECONDS); |
| } catch (final InterruptedException e) { |
| // empty |
| } |
| } |
| } |
| }; |
| |
| public ParallelTestRunner() throws Exception { |
| suite(); |
| } |
| |
| private static PrintStream outputStream() { |
| final String reportFile = System.getProperty(TEST_JS_REPORT_FILE, ""); |
| PrintStream output = System.out; |
| |
| if (!reportFile.isEmpty()) { |
| try { |
| output = new PrintStream(new OutputStreamDelegator(System.out, new FileOutputStream(reportFile))); |
| } catch (final IOException e) { |
| System.err.println(e); |
| } |
| } |
| |
| return output; |
| } |
| |
| public static final class ScriptRunnable extends AbstractScriptRunnable implements Callable<ScriptRunnable.Result> { |
| private final Result result = new Result(); |
| |
| public class Result { |
| private boolean passed = true; |
| public String expected; |
| public String out; |
| public String err; |
| public Throwable exception; |
| |
| public ScriptRunnable getTest() { |
| return ScriptRunnable.this; |
| } |
| |
| public boolean passed() { |
| return passed; |
| } |
| |
| @Override |
| public String toString() { |
| return getTest().toString(); |
| } |
| } |
| |
| public ScriptRunnable(final String framework, final File testFile, final List<String> engineOptions, final Map<String, String> testOptions, final List<String> scriptArguments) { |
| super(framework, testFile, engineOptions, testOptions, scriptArguments); |
| } |
| |
| @Override |
| protected void log(final String msg) { |
| System.err.println(msg); |
| } |
| |
| @Override |
| protected void fail(final String message) { |
| throw new TestFailedError(message); |
| } |
| |
| @Override |
| protected void compile() throws IOException { |
| final ByteArrayOutputStream out = new ByteArrayOutputStream(); |
| final ByteArrayOutputStream err = new ByteArrayOutputStream(); |
| final List<String> args = getCompilerArgs(); |
| int errors; |
| try { |
| errors = evaluateScript(out, err, args.toArray(new String[args.size()])); |
| } catch (final AssertionError e) { |
| final PrintWriter writer = new PrintWriter(err); |
| e.printStackTrace(writer); |
| writer.flush(); |
| errors = 1; |
| } |
| if (errors != 0 || checkCompilerMsg) { |
| result.err = err.toString(); |
| if (expectCompileFailure || checkCompilerMsg) { |
| final PrintStream outputDest = new PrintStream(new FileOutputStream(getErrorFileName())); |
| TestHelper.dumpFile(outputDest, new StringReader(new String(err.toByteArray()))); |
| outputDest.println("--"); |
| } |
| if (errors != 0 && !expectCompileFailure) { |
| fail(String.format("%d errors compiling %s", errors, testFile)); |
| } |
| if (checkCompilerMsg) { |
| compare(getErrorFileName(), expectedFileName, true); |
| } |
| } |
| if (expectCompileFailure && errors == 0) { |
| fail(String.format("No errors encountered compiling negative test %s", testFile)); |
| } |
| } |
| |
| @Override |
| protected void execute() { |
| final List<String> args = getRuntimeArgs(); |
| final ByteArrayOutputStream out = new ByteArrayOutputStream(); |
| final ByteArrayOutputStream err = new ByteArrayOutputStream(); |
| |
| try { |
| final int errors = evaluateScript(out, err, args.toArray(new String[args.size()])); |
| |
| if (errors != 0 || err.size() > 0) { |
| if (expectRunFailure) { |
| return; |
| } |
| if (!ignoreStdError) { |
| |
| try (OutputStream outputFile = new FileOutputStream(getOutputFileName()); OutputStream errorFile = new FileOutputStream(getErrorFileName())) { |
| outputFile.write(out.toByteArray()); |
| errorFile.write(err.toByteArray()); |
| } |
| |
| result.out = out.toString(); |
| result.err = err.toString(); |
| fail(err.toString()); |
| } |
| } |
| |
| if (compare) { |
| final File expectedFile = new File(expectedFileName); |
| try { |
| BufferedReader expected; |
| if (expectedFile.exists()) { |
| expected = new BufferedReader(new FileReader(expectedFile)); |
| } else { |
| expected = new BufferedReader(new StringReader("")); |
| } |
| compare(new BufferedReader(new StringReader(out.toString())), expected, false); |
| } catch (final Throwable ex) { |
| if (expectedFile.exists()) { |
| copyExpectedFile(); |
| } |
| try (OutputStream outputFile = new FileOutputStream(getOutputFileName()); OutputStream errorFile = new FileOutputStream(getErrorFileName())) { |
| outputFile.write(out.toByteArray()); |
| errorFile.write(err.toByteArray()); |
| } |
| ex.printStackTrace(); |
| throw ex; |
| } |
| } |
| } catch (final IOException e) { |
| if (!expectRunFailure) { |
| fail("Failure running test " + testFile + ": " + e.getMessage()); |
| } // else success |
| } |
| } |
| |
| private void compare(final String fileName, final String expected, final boolean compareCompilerMsg) throws IOException { |
| final File expectedFile = new File(expected); |
| |
| BufferedReader expectedReader; |
| if (expectedFile.exists()) { |
| expectedReader = new BufferedReader(new InputStreamReader(new FileInputStream(expectedFileName))); |
| } else { |
| expectedReader = new BufferedReader(new StringReader("")); |
| } |
| |
| final BufferedReader actual = new BufferedReader(new InputStreamReader(new FileInputStream(fileName))); |
| |
| compare(actual, expectedReader, compareCompilerMsg); |
| } |
| |
| private void copyExpectedFile() { |
| if (!new File(expectedFileName).exists()) { |
| return; |
| } |
| // copy expected file overwriting existing file and preserving last |
| // modified time of source |
| try { |
| Files.copy(FileSystems.getDefault().getPath(expectedFileName), FileSystems.getDefault().getPath(getCopyExpectedFileName()), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); |
| } catch (final IOException ex) { |
| fail("failed to copy expected " + expectedFileName + " to " + getCopyExpectedFileName() + ": " + ex.getMessage()); |
| } |
| } |
| |
| @Override |
| public Result call() { |
| try { |
| runTest(); |
| } catch (final Throwable ex) { |
| result.exception = ex; |
| result.passed = false; |
| ex.printStackTrace(); |
| } |
| return result; |
| } |
| |
| private String getOutputFileName() { |
| buildDir.mkdirs(); |
| return outputFileName; |
| } |
| |
| private String getErrorFileName() { |
| buildDir.mkdirs(); |
| return errorFileName; |
| } |
| |
| private String getCopyExpectedFileName() { |
| buildDir.mkdirs(); |
| return copyExpectedFileName; |
| } |
| } |
| |
| private void suite() throws Exception { |
| Locale.setDefault(new Locale("")); |
| System.setOut(outputStream()); |
| |
| final TestFactory<ScriptRunnable> testFactory = new TestFactory<ScriptRunnable>() { |
| @Override |
| public ScriptRunnable createTest(final String framework, final File testFile, final List<String> engineOptions, final Map<String, String> testOptions, final List<String> arguments) { |
| return new ScriptRunnable(framework, testFile, engineOptions, testOptions, arguments); |
| } |
| |
| @Override |
| public void log(final String msg) { |
| System.err.println(msg); |
| } |
| }; |
| |
| TestFinder.findAllTests(tests, orphans, testFactory); |
| |
| Collections.sort(tests, new Comparator<ScriptRunnable>() { |
| @Override |
| public int compare(final ScriptRunnable o1, final ScriptRunnable o2) { |
| return o1.testFile.compareTo(o2.testFile); |
| } |
| }); |
| } |
| |
| @SuppressWarnings("resource") |
| public boolean run() throws IOException { |
| final int testCount = tests.size(); |
| int passCount = 0; |
| int doneCount = 0; |
| System.out.printf("Found %d tests.\n", testCount); |
| final long startTime = System.nanoTime(); |
| |
| Runtime.getRuntime().addShutdownHook(shutdownHook); |
| |
| final List<Future<ScriptRunnable.Result>> futures = new ArrayList<>(); |
| for (final ScriptRunnable test : tests) { |
| futures.add(executor.submit(test)); |
| } |
| |
| executor.shutdown(); |
| try { |
| executor.awaitTermination(60, TimeUnit.MINUTES); |
| } catch (final InterruptedException ex) { |
| // empty |
| } |
| |
| final List<ScriptRunnable.Result> results = new ArrayList<>(); |
| for (final Future<ScriptRunnable.Result> future : futures) { |
| if (future.isDone()) { |
| try { |
| final ScriptRunnable.Result result = future.get(); |
| results.add(result); |
| doneCount++; |
| if (result.passed()) { |
| passCount++; |
| } |
| } catch (CancellationException | ExecutionException ex) { |
| ex.printStackTrace(); |
| } catch (final InterruptedException ex) { |
| assert false : "should not reach here"; |
| } |
| } |
| } |
| |
| Collections.sort(results, new Comparator<ScriptRunnable.Result>() { |
| @Override |
| public int compare(final ScriptRunnable.Result o1, final ScriptRunnable.Result o2) { |
| return o1.getTest().testFile.compareTo(o2.getTest().testFile); |
| } |
| }); |
| |
| boolean hasFailed = false; |
| final String failedList = System.getProperty(TEST_FAILED_LIST_FILE); |
| final boolean hasFailedList = failedList != null; |
| final boolean hadPreviouslyFailingTests = hasFailedList && new File(failedList).length() > 0; |
| final FileWriter failedFileWriter = hasFailedList ? new FileWriter(failedList) : null; |
| try { |
| final PrintWriter failedListWriter = failedFileWriter == null ? null : new PrintWriter(failedFileWriter); |
| for (final ScriptRunnable.Result result : results) { |
| if (!result.passed()) { |
| if (hasFailed == false) { |
| hasFailed = true; |
| System.out.println(); |
| System.out.println("FAILED TESTS"); |
| } |
| |
| System.out.println(result.getTest()); |
| if(failedFileWriter != null) { |
| failedListWriter.println(result.getTest().testFile.getPath()); |
| } |
| if (result.exception != null) { |
| final String exceptionString = result.exception instanceof TestFailedError ? result.exception.getMessage() : result.exception.toString(); |
| System.out.print(exceptionString.endsWith("\n") ? exceptionString : exceptionString + "\n"); |
| System.out.print(result.out != null ? result.out : ""); |
| } |
| } |
| } |
| } finally { |
| if(failedFileWriter != null) { |
| failedFileWriter.close(); |
| } |
| } |
| final double timeElapsed = (System.nanoTime() - startTime) / 1e9; // [s] |
| System.out.printf("Tests run: %d/%d tests, passed: %d (%.2f%%), failed: %d. Time elapsed: %.0fmin %.0fs.\n", doneCount, testCount, passCount, 100d * passCount / doneCount, doneCount - passCount, timeElapsed / 60, timeElapsed % 60); |
| System.out.flush(); |
| |
| finishedLatch.countDown(); |
| |
| if (hasFailed) { |
| throw new AssertionError("TEST FAILED"); |
| } |
| |
| if(hasFailedList) { |
| new File(failedList).delete(); |
| } |
| |
| if(hadPreviouslyFailingTests) { |
| System.out.println(); |
| System.out.println("Good job on getting all your previously failing tests pass!"); |
| System.out.println("NOW re-running all tests to make sure you haven't caused any NEW test failures."); |
| System.out.println(); |
| } |
| |
| return hadPreviouslyFailingTests; |
| } |
| |
| public static void main(final String[] args) throws Exception { |
| parseArgs(args); |
| |
| while (new ParallelTestRunner().run()) { |
| //empty |
| } |
| } |
| |
| private static void parseArgs(final String[] args) { |
| if (args.length > 0) { |
| String roots = ""; |
| String reportFile = ""; |
| for (int i = 0; i < args.length; i++) { |
| if (args[i].equals("--roots") && i != args.length - 1) { |
| roots += args[++i] + " "; |
| } else if (args[i].equals("--report-file") && i != args.length - 1) { |
| reportFile = args[++i]; |
| } else if (args[i].equals("--test262")) { |
| try { |
| setTest262Properties(); |
| } catch (final IOException ex) { |
| System.err.println(ex); |
| } |
| } |
| } |
| if (!roots.isEmpty()) { |
| System.setProperty(TEST_JS_ROOTS, roots.trim()); |
| } |
| if (!reportFile.isEmpty()) { |
| System.setProperty(TEST_JS_REPORT_FILE, reportFile); |
| } |
| } |
| } |
| |
| private static void setTest262Properties() throws IOException { |
| System.setProperty(TEST_JS_ROOTS, "test/test262/test/suite/"); |
| System.setProperty(TEST_JS_FRAMEWORK, "test/script/test262.js test/test262/test/harness/framework.js test/test262/test/harness/sta.js"); |
| System.setProperty(TEST_JS_EXCLUDES_FILE, "test/test262/test/config/excludelist.xml"); |
| System.setProperty(TEST_JS_ENABLE_STRICT_MODE, "true"); |
| |
| final Properties projectProperties = new Properties(); |
| projectProperties.load(new FileInputStream("project.properties")); |
| String excludeList = projectProperties.getProperty("test262-test-sys-prop.test.js.exclude.list", ""); |
| final Pattern pattern = Pattern.compile("\\$\\{([^}]+)}"); |
| for (;;) { |
| final Matcher matcher = pattern.matcher(excludeList); |
| if (!matcher.find()) { |
| break; |
| } |
| final String propertyValue = projectProperties.getProperty(matcher.group(1), ""); |
| excludeList = excludeList.substring(0, matcher.start()) + propertyValue + excludeList.substring(matcher.end()); |
| } |
| System.setProperty(TEST_JS_EXCLUDE_LIST, excludeList); |
| } |
| |
| public static final class OutputStreamDelegator extends OutputStream { |
| private final OutputStream[] streams; |
| |
| public OutputStreamDelegator(final OutputStream... streams) { |
| this.streams = streams; |
| } |
| |
| @Override |
| public void write(final int b) throws IOException { |
| for (final OutputStream stream : streams) { |
| stream.write(b); |
| } |
| } |
| |
| @Override |
| public void flush() throws IOException { |
| for (final OutputStream stream : streams) { |
| stream.flush(); |
| } |
| } |
| } |
| } |
| |
| final class TestFailedError extends Error { |
| private static final long serialVersionUID = 1L; |
| |
| public TestFailedError(final String message) { |
| super(message); |
| } |
| |
| public TestFailedError(final String message, final Throwable cause) { |
| super(message, cause); |
| } |
| |
| public TestFailedError(final Throwable cause) { |
| super(cause); |
| } |
| } |