| #!/usr/bin/python2.4 |
| |
| """Run reliability tests using Android instrumentation. |
| |
| A test file consists of list web sites to test is needed as a parameter |
| |
| Usage: |
| run_reliability_tests.py path/to/url/list |
| """ |
| |
| import logging |
| import optparse |
| import os |
| import subprocess |
| import sys |
| import time |
| from Numeric import * |
| |
| TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt" |
| TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt" |
| TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt" |
| TEST_LOAD_TIME_FILE = "/sdcard/android/reliability_load_time.txt" |
| HTTP_URL_FILE = "urllist_http" |
| HTTPS_URL_FILE = "urllist_https" |
| NUM_URLS = 25 |
| |
| |
| def DumpRenderTreeFinished(adb_cmd): |
| """Check if DumpRenderTree finished running. |
| |
| Args: |
| adb_cmd: adb command string |
| |
| Returns: |
| True if DumpRenderTree has finished, False otherwise |
| """ |
| |
| # pull test status file and look for "#DONE" |
| shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE |
| adb_output = subprocess.Popen(shell_cmd_str, |
| shell=True, stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE).communicate()[0] |
| return adb_output.strip() == "#DONE" |
| |
| |
| def RemoveDeviceFile(adb_cmd, file_name): |
| shell_cmd_str = adb_cmd + " shell rm " + file_name |
| subprocess.Popen(shell_cmd_str, |
| shell=True, stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE).communicate() |
| |
| |
| def Bugreport(url, bugreport_dir, adb_cmd): |
| """Pull a bugreport from the device.""" |
| bugreport_filename = "%s/reliability_bugreport_%d.txt" % (bugreport_dir, |
| int(time.time())) |
| |
| # prepend the report with url |
| handle = open(bugreport_filename, "w") |
| handle.writelines("Bugreport for crash in url - %s\n\n" % url) |
| handle.close() |
| |
| cmd = "%s bugreport >> %s" % (adb_cmd, bugreport_filename) |
| os.system(cmd) |
| |
| |
| def ProcessPageLoadTime(raw_log): |
| """Processes the raw page load time logged by test app.""" |
| log_handle = open(raw_log, "r") |
| load_times = {} |
| |
| for line in log_handle: |
| line = line.strip() |
| pair = line.split("|") |
| if len(pair) != 2: |
| logging.info("Line has more than one '|': " + line) |
| continue |
| if pair[0] not in load_times: |
| load_times[pair[0]] = [] |
| try: |
| pair[1] = int(pair[1]) |
| except ValueError: |
| logging.info("Lins has non-numeric load time: " + line) |
| continue |
| load_times[pair[0]].append(pair[1]) |
| |
| log_handle.close() |
| |
| # rewrite the average time to file |
| log_handle = open(raw_log, "w") |
| for url, times in load_times.iteritems(): |
| # calculate std |
| arr = array(times) |
| avg = average(arr) |
| d = arr - avg |
| std = sqrt(sum(d * d) / len(arr)) |
| output = ("%-70s%-10d%-10d%-12.2f%-12.2f%s\n" % |
| (url, min(arr), max(arr), avg, std, |
| array2string(arr))) |
| log_handle.write(output) |
| log_handle.close() |
| |
| |
| def main(options, args): |
| """Send the url list to device and start testing, restart if crashed.""" |
| |
| # Set up logging format. |
| log_level = logging.INFO |
| if options.verbose: |
| log_level = logging.DEBUG |
| logging.basicConfig(level=log_level, |
| format="%(message)s") |
| |
| # Include all tests if none are specified. |
| if not args: |
| print "Missing URL list file" |
| sys.exit(1) |
| else: |
| path = args[0] |
| |
| if not options.crash_file: |
| print "Missing crash file name, use --crash-file to specify" |
| sys.exit(1) |
| else: |
| crashed_file = options.crash_file |
| |
| if not options.timeout_file: |
| print "Missing timeout file, use --timeout-file to specify" |
| sys.exit(1) |
| else: |
| timedout_file = options.timeout_file |
| |
| if not options.delay: |
| manual_delay = 0 |
| else: |
| manual_delay = options.delay |
| |
| if not options.bugreport: |
| bugreport_dir = "." |
| else: |
| bugreport_dir = options.bugreport |
| if not os.path.exists(bugreport_dir): |
| os.makedirs(bugreport_dir) |
| if not os.path.isdir(bugreport_dir): |
| logging.error("Cannot create results dir: " + bugreport_dir) |
| sys.exit(1) |
| |
| adb_cmd = "adb " |
| if options.adb_options: |
| adb_cmd += options.adb_options + " " |
| |
| # push url list to device |
| test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\"" |
| proc = subprocess.Popen(test_cmd, shell=True, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE) |
| (adb_output, adb_error) = proc.communicate() |
| if proc.returncode != 0: |
| logging.error("failed to push url list to device.") |
| logging.error(adb_output) |
| logging.error(adb_error) |
| sys.exit(1) |
| |
| # clean up previous results |
| RemoveDeviceFile(adb_cmd, TEST_STATUS_FILE) |
| RemoveDeviceFile(adb_cmd, TEST_TIMEOUT_FILE) |
| RemoveDeviceFile(adb_cmd, TEST_LOAD_TIME_FILE) |
| |
| logging.info("Running the test ...") |
| |
| # Count crashed tests. |
| crashed_tests = [] |
| |
| if options.time_out_ms: |
| timeout_ms = options.time_out_ms |
| |
| # Run test until it's done |
| test_cmd_prefix = adb_cmd + " shell am instrument" |
| test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner" |
| |
| # Call ReliabilityTestsAutoTest#startReliabilityTests |
| test_cmd = (test_cmd_prefix + " -e class " |
| "com.android.dumprendertree.ReliabilityTest#" |
| "runReliabilityTest -e timeout %s -e delay %s" % |
| (str(timeout_ms), str(manual_delay))) |
| |
| if options.logtime: |
| test_cmd += " -e logtime true" |
| |
| test_cmd += test_cmd_postfix |
| |
| adb_output = subprocess.Popen(test_cmd, shell=True, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE).communicate()[0] |
| while not DumpRenderTreeFinished(adb_cmd): |
| logging.error("DumpRenderTree exited before all URLs are visited.") |
| shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE |
| crashed_test = subprocess.Popen(shell_cmd_str, shell=True, |
| stdout=subprocess.PIPE).communicate()[0] |
| logging.info(crashed_test + " CRASHED") |
| crashed_tests.append(crashed_test) |
| Bugreport(crashed_test, bugreport_dir, adb_cmd) |
| logging.info("Resuming reliability test runner...") |
| |
| adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE).communicate()[0] |
| |
| if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or |
| adb_output.find("Process crashed.") != -1): |
| logging.error("Error happened : " + adb_output) |
| sys.exit(1) |
| |
| logging.info(adb_output) |
| logging.info("Done\n") |
| |
| if crashed_tests: |
| file_handle = open(crashed_file, "w") |
| file_handle.writelines("\n".join(crashed_tests)) |
| logging.info("Crashed URL list stored in: " + crashed_file) |
| file_handle.close() |
| else: |
| logging.info("No crash found.") |
| |
| # get timeout file from sdcard |
| test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \"" |
| + timedout_file + "\"") |
| subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE).communicate() |
| |
| if options.logtime: |
| # get logged page load times from sdcard |
| test_cmd = (adb_cmd + "pull \"" + TEST_LOAD_TIME_FILE + "\" \"" |
| + options.logtime + "\"") |
| subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE).communicate() |
| ProcessPageLoadTime(options.logtime) |
| |
| |
| if "__main__" == __name__: |
| option_parser = optparse.OptionParser() |
| option_parser.add_option("-t", "--time-out-ms", |
| default=60000, |
| help="set the timeout for each test") |
| option_parser.add_option("-v", "--verbose", action="store_true", |
| default=False, |
| help="include debug-level logging") |
| option_parser.add_option("-a", "--adb-options", |
| default=None, |
| help="pass options to adb, such as -d -e, etc") |
| option_parser.add_option("-c", "--crash-file", |
| default="reliability_crashed_sites.txt", |
| help="the list of sites that cause browser to crash") |
| option_parser.add_option("-f", "--timeout-file", |
| default="reliability_timedout_sites.txt", |
| help="the list of sites that timedout during test") |
| option_parser.add_option("-d", "--delay", |
| default=0, |
| help="add a manual delay between pages (in ms)") |
| option_parser.add_option("-b", "--bugreport", |
| default=".", |
| help="the directory to store bugreport for crashes") |
| option_parser.add_option("-l", "--logtime", |
| default=None, |
| help="Logs page load time for each url to the file") |
| opts, arguments = option_parser.parse_args() |
| main(opts, arguments) |