tests/DumpRenderTree/assets/run_reliability_tests.py - platform/frameworks/base - Git at Google

 #!/usr/bin/python2.4

 """Run reliability tests using Android instrumentation.

   A test file consists of list web sites to test is needed as a parameter

   Usage:
     run_reliability_tests.py path/to/url/list
 """

 import logging
 import optparse
 import os
 import subprocess
 import sys
 import time
 from Numeric import *

 TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt"
 TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt"
 TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt"
 TEST_LOAD_TIME_FILE = "/sdcard/android/reliability_load_time.txt"
 HTTP_URL_FILE = "urllist_http"
 HTTPS_URL_FILE = "urllist_https"
 NUM_URLS = 25


 def DumpRenderTreeFinished(adb_cmd):
   """Check if DumpRenderTree finished running.

   Args:
     adb_cmd: adb command string

   Returns:
     True if DumpRenderTree has finished, False otherwise
   """

   # pull test status file and look for "#DONE"
   shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
   adb_output = subprocess.Popen(shell_cmd_str,
                                 shell=True, stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE).communicate()[0]
   return adb_output.strip() == "#DONE"


 def RemoveDeviceFile(adb_cmd, file_name):
   shell_cmd_str = adb_cmd + " shell rm " + file_name
   subprocess.Popen(shell_cmd_str,
                    shell=True, stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE).communicate()


 def Bugreport(url, bugreport_dir, adb_cmd):
   """Pull a bugreport from the device."""
   bugreport_filename = "%s/reliability_bugreport_%d.txt" % (bugreport_dir,
                                                             int(time.time()))

   # prepend the report with url
   handle = open(bugreport_filename, "w")
   handle.writelines("Bugreport for crash in url - %s\n\n" % url)
   handle.close()

   cmd = "%s bugreport >> %s" % (adb_cmd, bugreport_filename)
   os.system(cmd)


 def ProcessPageLoadTime(raw_log):
   """Processes the raw page load time logged by test app."""
   log_handle = open(raw_log, "r")
   load_times = {}

   for line in log_handle:
     line = line.strip()
     pair = line.split("|")
     if len(pair) != 2:
       logging.info("Line has more than one '|': " + line)
       continue
     if pair[0] not in load_times:
       load_times[pair[0]] = []
     try:
       pair[1] = int(pair[1])
     except ValueError:
       logging.info("Lins has non-numeric load time: " + line)
       continue
     load_times[pair[0]].append(pair[1])

   log_handle.close()

   # rewrite the average time to file
   log_handle = open(raw_log, "w")
   for url, times in load_times.iteritems():
     # calculate std
     arr = array(times)
     avg = average(arr)
     d = arr - avg
     std = sqrt(sum(d * d) / len(arr))
     output = ("%-70s%-10d%-10d%-12.2f%-12.2f%s\n" %
               (url, min(arr), max(arr), avg, std,
                array2string(arr)))
     log_handle.write(output)
   log_handle.close()


 def main(options, args):
   """Send the url list to device and start testing, restart if crashed."""

   # Set up logging format.
   log_level = logging.INFO
   if options.verbose:
     log_level = logging.DEBUG
   logging.basicConfig(level=log_level,
                       format="%(message)s")

   # Include all tests if none are specified.
   if not args:
     print "Missing URL list file"
     sys.exit(1)
   else:
     path = args[0]

   if not options.crash_file:
     print "Missing crash file name, use --crash-file to specify"
     sys.exit(1)
   else:
     crashed_file = options.crash_file

   if not options.timeout_file:
     print "Missing timeout file, use --timeout-file to specify"
     sys.exit(1)
   else:
     timedout_file = options.timeout_file

   if not options.delay:
     manual_delay = 0
   else:
     manual_delay = options.delay

   if not options.bugreport:
     bugreport_dir = "."
   else:
     bugreport_dir = options.bugreport
   if not os.path.exists(bugreport_dir):
     os.makedirs(bugreport_dir)
   if not os.path.isdir(bugreport_dir):
     logging.error("Cannot create results dir: " + bugreport_dir)
     sys.exit(1)

   adb_cmd = "adb "
   if options.adb_options:
     adb_cmd += options.adb_options + " "

   # push url list to device
   test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\""
   proc = subprocess.Popen(test_cmd, shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
   (adb_output, adb_error) = proc.communicate()
   if proc.returncode != 0:
     logging.error("failed to push url list to device.")
     logging.error(adb_output)
     logging.error(adb_error)
     sys.exit(1)

   # clean up previous results
   RemoveDeviceFile(adb_cmd, TEST_STATUS_FILE)
   RemoveDeviceFile(adb_cmd, TEST_TIMEOUT_FILE)
   RemoveDeviceFile(adb_cmd, TEST_LOAD_TIME_FILE)

   logging.info("Running the test ...")

   # Count crashed tests.
   crashed_tests = []

   if options.time_out_ms:
     timeout_ms = options.time_out_ms

   # Run test until it's done
   test_cmd_prefix = adb_cmd + " shell am instrument"
   test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner"

   # Call ReliabilityTestsAutoTest#startReliabilityTests
   test_cmd = (test_cmd_prefix + " -e class "
               "com.android.dumprendertree.ReliabilityTest#"
               "runReliabilityTest -e timeout %s -e delay %s" %
               (str(timeout_ms), str(manual_delay)))

   if options.logtime:
     test_cmd += " -e logtime true"

   test_cmd += test_cmd_postfix

   adb_output = subprocess.Popen(test_cmd, shell=True,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE).communicate()[0]
   while not DumpRenderTreeFinished(adb_cmd):
     logging.error("DumpRenderTree exited before all URLs are visited.")
     shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
     crashed_test = ""
     while not crashed_test:
       (crashed_test, err) = subprocess.Popen(
           shell_cmd_str, shell=True, stdout=subprocess.PIPE,
           stderr=subprocess.PIPE).communicate()
       crashed_test = crashed_test.strip()
       if not crashed_test:
         logging.error('Cannot get crashed test name, device offline?')
         logging.error('stderr: ' + err)
         logging.error('retrying in 10s...')
         time.sleep(10)

     logging.info(crashed_test + " CRASHED")
     crashed_tests.append(crashed_test)
     Bugreport(crashed_test, bugreport_dir, adb_cmd)
     logging.info("Resuming reliability test runner...")

     adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE).communicate()[0]

   if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or
       adb_output.find("Process crashed.") != -1):
     logging.error("Error happened : " + adb_output)
     sys.exit(1)

   logging.info(adb_output)
   logging.info("Done\n")

   if crashed_tests:
     file_handle = open(crashed_file, "w")
     file_handle.writelines("\n".join(crashed_tests))
     logging.info("Crashed URL list stored in: " + crashed_file)
     file_handle.close()
   else:
     logging.info("No crash found.")

   # get timeout file from sdcard
   test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \""
               + timedout_file +  "\"")
   subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE).communicate()

   if options.logtime:
     # get logged page load times from sdcard
     test_cmd = (adb_cmd + "pull \"" + TEST_LOAD_TIME_FILE + "\" \""
                 + options.logtime +  "\"")
     subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
                      stderr=subprocess.PIPE).communicate()
     ProcessPageLoadTime(options.logtime)


 if "__main__" == __name__:
   option_parser = optparse.OptionParser()
   option_parser.add_option("-t", "--time-out-ms",
                            default=60000,
                            help="set the timeout for each test")
   option_parser.add_option("-v", "--verbose", action="store_true",
                            default=False,
                            help="include debug-level logging")
   option_parser.add_option("-a", "--adb-options",
                            default=None,
                            help="pass options to adb, such as -d -e, etc")
   option_parser.add_option("-c", "--crash-file",
                            default="reliability_crashed_sites.txt",
                            help="the list of sites that cause browser to crash")
   option_parser.add_option("-f", "--timeout-file",
                            default="reliability_timedout_sites.txt",
                            help="the list of sites that timedout during test")
   option_parser.add_option("-d", "--delay",
                            default=0,
                            help="add a manual delay between pages (in ms)")
   option_parser.add_option("-b", "--bugreport",
                            default=".",
                            help="the directory to store bugreport for crashes")
   option_parser.add_option("-l", "--logtime",
                            default=None,
                            help="Logs page load time for each url to the file")
   opts, arguments = option_parser.parse_args()
   main(opts, arguments)
	#!/usr/bin/python2.4

	"""Run reliability tests using Android instrumentation.

	A test file consists of list web sites to test is needed as a parameter

	Usage:
	run_reliability_tests.py path/to/url/list
	"""

	import logging
	import optparse
	import os
	import subprocess
	import sys
	import time
	from Numeric import *

	TEST_LIST_FILE = "/sdcard/android/reliability_tests_list.txt"
	TEST_STATUS_FILE = "/sdcard/android/reliability_running_test.txt"
	TEST_TIMEOUT_FILE = "/sdcard/android/reliability_timeout_test.txt"
	TEST_LOAD_TIME_FILE = "/sdcard/android/reliability_load_time.txt"
	HTTP_URL_FILE = "urllist_http"
	HTTPS_URL_FILE = "urllist_https"
	NUM_URLS = 25


	def DumpRenderTreeFinished(adb_cmd):
	"""Check if DumpRenderTree finished running.

	Args:
	adb_cmd: adb command string

	Returns:
	True if DumpRenderTree has finished, False otherwise
	"""

	# pull test status file and look for "#DONE"
	shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
	adb_output = subprocess.Popen(shell_cmd_str,
	shell=True, stdout=subprocess.PIPE,
	stderr=subprocess.PIPE).communicate()[0]
	return adb_output.strip() == "#DONE"


	def RemoveDeviceFile(adb_cmd, file_name):
	shell_cmd_str = adb_cmd + " shell rm " + file_name
	subprocess.Popen(shell_cmd_str,
	shell=True, stdout=subprocess.PIPE,
	stderr=subprocess.PIPE).communicate()


	def Bugreport(url, bugreport_dir, adb_cmd):
	"""Pull a bugreport from the device."""
	bugreport_filename = "%s/reliability_bugreport_%d.txt" % (bugreport_dir,
	int(time.time()))

	# prepend the report with url
	handle = open(bugreport_filename, "w")
	handle.writelines("Bugreport for crash in url - %s\n\n" % url)
	handle.close()

	cmd = "%s bugreport >> %s" % (adb_cmd, bugreport_filename)
	os.system(cmd)


	def ProcessPageLoadTime(raw_log):
	"""Processes the raw page load time logged by test app."""
	log_handle = open(raw_log, "r")
	load_times = {}

	for line in log_handle:
	line = line.strip()
	pair = line.split("\|")
	if len(pair) != 2:
	logging.info("Line has more than one '\|': " + line)
	continue
	if pair[0] not in load_times:
	load_times[pair[0]] = []
	try:
	pair[1] = int(pair[1])
	except ValueError:
	logging.info("Lins has non-numeric load time: " + line)
	continue
	load_times[pair[0]].append(pair[1])

	log_handle.close()

	# rewrite the average time to file
	log_handle = open(raw_log, "w")
	for url, times in load_times.iteritems():
	# calculate std
	arr = array(times)
	avg = average(arr)
	d = arr - avg
	std = sqrt(sum(d * d) / len(arr))
	output = ("%-70s%-10d%-10d%-12.2f%-12.2f%s\n" %
	(url, min(arr), max(arr), avg, std,
	array2string(arr)))
	log_handle.write(output)
	log_handle.close()


	def main(options, args):
	"""Send the url list to device and start testing, restart if crashed."""

	# Set up logging format.
	log_level = logging.INFO
	if options.verbose:
	log_level = logging.DEBUG
	logging.basicConfig(level=log_level,
	format="%(message)s")

	# Include all tests if none are specified.
	if not args:
	print "Missing URL list file"
	sys.exit(1)
	else:
	path = args[0]

	if not options.crash_file:
	print "Missing crash file name, use --crash-file to specify"
	sys.exit(1)
	else:
	crashed_file = options.crash_file

	if not options.timeout_file:
	print "Missing timeout file, use --timeout-file to specify"
	sys.exit(1)
	else:
	timedout_file = options.timeout_file

	if not options.delay:
	manual_delay = 0
	else:
	manual_delay = options.delay

	if not options.bugreport:
	bugreport_dir = "."
	else:
	bugreport_dir = options.bugreport
	if not os.path.exists(bugreport_dir):
	os.makedirs(bugreport_dir)
	if not os.path.isdir(bugreport_dir):
	logging.error("Cannot create results dir: " + bugreport_dir)
	sys.exit(1)

	adb_cmd = "adb "
	if options.adb_options:
	adb_cmd += options.adb_options + " "

	# push url list to device
	test_cmd = adb_cmd + " push \"" + path + "\" \"" + TEST_LIST_FILE + "\""
	proc = subprocess.Popen(test_cmd, shell=True,
	stdout=subprocess.PIPE,
	stderr=subprocess.PIPE)
	(adb_output, adb_error) = proc.communicate()
	if proc.returncode != 0:
	logging.error("failed to push url list to device.")
	logging.error(adb_output)
	logging.error(adb_error)
	sys.exit(1)

	# clean up previous results
	RemoveDeviceFile(adb_cmd, TEST_STATUS_FILE)
	RemoveDeviceFile(adb_cmd, TEST_TIMEOUT_FILE)
	RemoveDeviceFile(adb_cmd, TEST_LOAD_TIME_FILE)

	logging.info("Running the test ...")

	# Count crashed tests.
	crashed_tests = []

	if options.time_out_ms:
	timeout_ms = options.time_out_ms

	# Run test until it's done
	test_cmd_prefix = adb_cmd + " shell am instrument"
	test_cmd_postfix = " -w com.android.dumprendertree/.LayoutTestsAutoRunner"

	# Call ReliabilityTestsAutoTest#startReliabilityTests
	test_cmd = (test_cmd_prefix + " -e class "
	"com.android.dumprendertree.ReliabilityTest#"
	"runReliabilityTest -e timeout %s -e delay %s" %
	(str(timeout_ms), str(manual_delay)))

	if options.logtime:
	test_cmd += " -e logtime true"

	test_cmd += test_cmd_postfix

	adb_output = subprocess.Popen(test_cmd, shell=True,
	stdout=subprocess.PIPE,
	stderr=subprocess.PIPE).communicate()[0]
	while not DumpRenderTreeFinished(adb_cmd):
	logging.error("DumpRenderTree exited before all URLs are visited.")
	shell_cmd_str = adb_cmd + " shell cat " + TEST_STATUS_FILE
	crashed_test = ""
	while not crashed_test:
	(crashed_test, err) = subprocess.Popen(
	shell_cmd_str, shell=True, stdout=subprocess.PIPE,
	stderr=subprocess.PIPE).communicate()
	crashed_test = crashed_test.strip()
	if not crashed_test:
	logging.error('Cannot get crashed test name, device offline?')
	logging.error('stderr: ' + err)
	logging.error('retrying in 10s...')
	time.sleep(10)

	logging.info(crashed_test + " CRASHED")
	crashed_tests.append(crashed_test)
	Bugreport(crashed_test, bugreport_dir, adb_cmd)
	logging.info("Resuming reliability test runner...")

	adb_output = subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
	stderr=subprocess.PIPE).communicate()[0]

	if (adb_output.find("INSTRUMENTATION_FAILED") != -1 or
	adb_output.find("Process crashed.") != -1):
	logging.error("Error happened : " + adb_output)
	sys.exit(1)

	logging.info(adb_output)
	logging.info("Done\n")

	if crashed_tests:
	file_handle = open(crashed_file, "w")
	file_handle.writelines("\n".join(crashed_tests))
	logging.info("Crashed URL list stored in: " + crashed_file)
	file_handle.close()
	else:
	logging.info("No crash found.")

	# get timeout file from sdcard
	test_cmd = (adb_cmd + "pull \"" + TEST_TIMEOUT_FILE + "\" \""
	+ timedout_file + "\"")
	subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
	stderr=subprocess.PIPE).communicate()

	if options.logtime:
	# get logged page load times from sdcard
	test_cmd = (adb_cmd + "pull \"" + TEST_LOAD_TIME_FILE + "\" \""
	+ options.logtime + "\"")
	subprocess.Popen(test_cmd, shell=True, stdout=subprocess.PIPE,
	stderr=subprocess.PIPE).communicate()
	ProcessPageLoadTime(options.logtime)


	if "__main__" == __name__:
	option_parser = optparse.OptionParser()
	option_parser.add_option("-t", "--time-out-ms",
	default=60000,
	help="set the timeout for each test")
	option_parser.add_option("-v", "--verbose", action="store_true",
	default=False,
	help="include debug-level logging")
	option_parser.add_option("-a", "--adb-options",
	default=None,
	help="pass options to adb, such as -d -e, etc")
	option_parser.add_option("-c", "--crash-file",
	default="reliability_crashed_sites.txt",
	help="the list of sites that cause browser to crash")
	option_parser.add_option("-f", "--timeout-file",
	default="reliability_timedout_sites.txt",
	help="the list of sites that timedout during test")
	option_parser.add_option("-d", "--delay",
	default=0,
	help="add a manual delay between pages (in ms)")
	option_parser.add_option("-b", "--bugreport",
	default=".",
	help="the directory to store bugreport for crashes")
	option_parser.add_option("-l", "--logtime",
	default=None,
	help="Logs page load time for each url to the file")
	opts, arguments = option_parser.parse_args()
	main(opts, arguments)