Prepare run_jfuzz_test.py to report bugs
Adds --report_script and --fuzz_arg switches.
Report script is called for every divergence with title of
divergence, a comment and potentially bisection search output file.
Fuzz args are passed to jfuzz. They can be used to reproduce
previously discovered divergence.
Also add a -v switch to jfuzz. If present jfuzz will print its
version and exit.
Test: ./tools/jfuzz/run_jfuzz_test.py --report_script
$HOME/report_issue.py --fuzz_arg='-s 1470736838'
Change-Id: I25fd305304edfe21071a81d6e1b1b47ae8703007
diff --git a/tools/jfuzz/README.md b/tools/jfuzz/README.md
index 1d566a9..0f53d93 100644
--- a/tools/jfuzz/README.md
+++ b/tools/jfuzz/README.md
@@ -14,7 +14,7 @@
===================
jfuzz [-s seed] [-d expr-depth] [-l stmt-length]
- [-i if-nest] [-n loop-nest]
+ [-i if-nest] [-n loop-nest] [-v] [-h]
where
@@ -28,6 +28,8 @@
(higher values yield deeper nested conditionals)
-n : defines a fuzzing nest for for/while/do-while loops
(higher values yield deeper nested loops)
+ -v : prints version number and exits
+ -h : prints help and exits
The current version of JFuzz sends all output to stdout, and uses
a fixed testing class named Test. So a typical test run looks as follows.
@@ -43,18 +45,22 @@
[--num_tests=NUM_TESTS]
[--device=DEVICE]
[--mode1=MODE] [--mode2=MODE]
+ [--report_script=SCRIPT]
+ [--jfuzz_arg=ARG]
where
- --num_tests : number of tests to run (10000 by default)
- --device : target device serial number (passed to adb -s)
- --mode1 : m1
- --mode2 : m2, with m1 != m2, and values one of
+ --num_tests : number of tests to run (10000 by default)
+ --device : target device serial number (passed to adb -s)
+ --mode1 : m1
+ --mode2 : m2, with m1 != m2, and values one of
ri = reference implementation on host (default for m1)
hint = Art interpreter on host
hopt = Art optimizing on host (default for m2)
tint = Art interpreter on target
topt = Art optimizing on target
+ --report_script : path to script called for each divergence
+ --jfuzz_arg : argument for jfuzz
How to start J/DexFuzz testing (multi-layered)
==============================================
diff --git a/tools/jfuzz/jfuzz.cc b/tools/jfuzz/jfuzz.cc
index 125b56a..88205c8 100644
--- a/tools/jfuzz/jfuzz.cc
+++ b/tools/jfuzz/jfuzz.cc
@@ -1067,7 +1067,7 @@
// Parse options.
while (1) {
- int32_t option = getopt(argc, argv, "s:d:l:i:n:h");
+ int32_t option = getopt(argc, argv, "s:d:l:i:n:vh");
if (option < 0) {
break; // done
}
@@ -1087,12 +1087,15 @@
case 'n':
loop_nest = strtoul(optarg, nullptr, 0);
break;
+ case 'v':
+ fprintf(stderr, "jfuzz version %s\n", VERSION);
+ return 0;
case 'h':
default:
fprintf(stderr,
"usage: %s [-s seed] "
"[-d expr-depth] [-l stmt-length] "
- "[-i if-nest] [-n loop-nest] [-h]\n",
+ "[-i if-nest] [-n loop-nest] [-v] [-h]\n",
argv[0]);
return 1;
}
diff --git a/tools/jfuzz/run_jfuzz_test.py b/tools/jfuzz/run_jfuzz_test.py
index cf2364b..05e8daf 100755
--- a/tools/jfuzz/run_jfuzz_test.py
+++ b/tools/jfuzz/run_jfuzz_test.py
@@ -20,9 +20,11 @@
import os
import shlex
import shutil
+import subprocess
import sys
from glob import glob
+from subprocess import DEVNULL
from tempfile import mkdtemp
sys.path.append(os.path.dirname(os.path.dirname(
@@ -34,6 +36,7 @@
from common.common import GetJackClassPath
from common.common import GetEnvVariableOrError
from common.common import RunCommand
+from common.common import RunCommandForOutput
from common.common import DeviceTestEnv
# Return codes supported by bisection bug search.
@@ -302,7 +305,8 @@
class JFuzzTester(object):
"""Tester that runs JFuzz many times and report divergences."""
- def __init__(self, num_tests, device, mode1, mode2):
+ def __init__(self, num_tests, device, mode1, mode2, jfuzz_args,
+ report_script):
"""Constructor for the tester.
Args:
@@ -310,11 +314,15 @@
device: string, target device serial number (or None)
mode1: string, execution mode for first runner
mode2: string, execution mode for second runner
+ jfuzz_args: list of strings, additional arguments for jfuzz
+ report_script: string, path to script called for each divergence
"""
self._num_tests = num_tests
self._device = device
self._runner1 = GetExecutionModeRunner(device, mode1)
self._runner2 = GetExecutionModeRunner(device, mode2)
+ self._jfuzz_args = jfuzz_args
+ self._report_script = report_script
self._save_dir = None
self._results_dir = None
self._jfuzz_dir = None
@@ -392,7 +400,8 @@
Raises:
FatalError: error when jfuzz fails
"""
- if RunCommand(['jfuzz'], out='Test.java', err=None) != RetCode.SUCCESS:
+ if (RunCommand(['jfuzz'] + self._jfuzz_args, out='Test.java', err=None)
+ != RetCode.SUCCESS):
raise FatalError('Unexpected error while running JFuzz')
def CheckForDivergence(self, retc1, retc2):
@@ -442,6 +451,44 @@
# Maybe run bisection bug search.
if retc1 in BISECTABLE_RET_CODES and retc2 in BISECTABLE_RET_CODES:
self.MaybeBisectDivergence(retc1, retc2, is_output_divergence)
+ # Call reporting script.
+ if self._report_script:
+ self.RunReportScript(retc1, retc2, is_output_divergence)
+
+ def RunReportScript(self, retc1, retc2, is_output_divergence):
+ """Runs report script."""
+ try:
+ title = "Divergence between {0} and {1} (found with fuzz testing)".format(
+ self._runner1.description, self._runner2.description)
+ # Prepare divergence comment.
+ jfuzz_cmd_and_version = subprocess.check_output(
+ ['grep', '-o', 'jfuzz.*', 'Test.java'], universal_newlines=True)
+ (jfuzz_cmd_str, jfuzz_ver) = jfuzz_cmd_and_version.split('(')
+ # Strip right parenthesis and new line.
+ jfuzz_ver = jfuzz_ver[:-2]
+ jfuzz_args = ['\'-{0}\''.format(arg)
+ for arg in jfuzz_cmd_str.strip().split(' -')][1:]
+ wrapped_args = ['--jfuzz_arg={0}'.format(opt) for opt in jfuzz_args]
+ repro_cmd_str = (os.path.basename(__file__) + ' --num_tests 1 ' +
+ ' '.join(wrapped_args))
+ comment = 'jfuzz {0}\nReproduce test:\n{1}\nReproduce divergence:\n{2}\n'.format(
+ jfuzz_ver, jfuzz_cmd_str, repro_cmd_str)
+ if is_output_divergence:
+ (output, _, _) = RunCommandForOutput(
+ ['diff', self._runner1.output_file, self._runner2.output_file],
+ None, subprocess.PIPE, subprocess.STDOUT)
+ comment += 'Diff:\n' + output
+ else:
+ comment += '{0} vs {1}\n'.format(retc1, retc2)
+ # Prepare report script command.
+ script_cmd = [self._report_script, title, comment]
+ ddir = self.GetCurrentDivergenceDir()
+ bisection_out_files = glob(ddir + '/*_bisection_out.txt')
+ if bisection_out_files:
+ script_cmd += ['--bisection_out', bisection_out_files[0]]
+ subprocess.check_call(script_cmd, stdout=DEVNULL, stderr=DEVNULL)
+ except subprocess.CalledProcessError as err:
+ print('Failed to run report script.\n', err)
def RunBisectionSearch(self, args, expected_retcode, expected_output,
runner_id):
@@ -495,12 +542,16 @@
help='execution mode 1 (default: ri)')
parser.add_argument('--mode2', default='hopt',
help='execution mode 2 (default: hopt)')
+ parser.add_argument('--report_script', help='script called for each'
+ 'divergence')
+ parser.add_argument('--jfuzz_arg', default=[], dest='jfuzz_args',
+ action='append', help='argument for jfuzz')
args = parser.parse_args()
if args.mode1 == args.mode2:
raise FatalError('Identical execution modes given')
# Run the JFuzz tester.
- with JFuzzTester(args.num_tests, args.device,
- args.mode1, args.mode2) as fuzzer:
+ with JFuzzTester(args.num_tests, args.device, args.mode1, args.mode2,
+ args.jfuzz_args, args.report_script) as fuzzer:
fuzzer.Run()
if __name__ == '__main__':
