Timeout the test run before go/ab timeout to capture test info
go/ab timeouts in 14500 secs. The testrunner will now have a global timeout
to end all the test runs before go/ab timeout so to capture test info. It
also prints out time the duration for each test if timeout is hit.
Test: ./art/test/testrunner/testrunner.py
Change-Id: If29d2dbe69b8117d9ebf848c582451c0ad39711a
diff --git a/test/testrunner/testrunner.py b/test/testrunner/testrunner.py
index 68e1856..2c64b56 100755
--- a/test/testrunner/testrunner.py
+++ b/test/testrunner/testrunner.py
@@ -50,6 +50,7 @@
import json
import multiprocessing
import os
+import operator
import re
import subprocess
import sys
@@ -75,9 +76,11 @@
OPTIMIZING_COMPILER_TYPES = set()
JVMTI_TYPES = set()
ADDRESS_SIZES_TARGET = {'host': set(), 'target': set()}
+TIME_STATS = {}
# timeout for individual tests.
# TODO: make it adjustable per tests and for buildbots
timeout = 3000 # 50 minutes
+global_timeout = 14100 # 235 minutes (The go/ab timeout is 14500)
# DISABLED_TEST_CONTAINER holds information about the disabled tests. It is a map
# that has key as the test name (like 001-HelloWorld), and value as set of
@@ -355,7 +358,7 @@
# stops creating any any thread and wait for all the exising threads
# to end.
while threading.active_count() > 2:
- time.sleep(0.1)
+ time.sleep(1)
return
test_name = 'test-art-'
test_name += target + '-run-test-'
@@ -506,11 +509,13 @@
test_skipped = True
else:
test_skipped = False
+ start_recording_time(test_name)
if gdb:
proc = subprocess.Popen(command.split(), stderr=subprocess.STDOUT, universal_newlines=True)
else:
proc = subprocess.Popen(command.split(), stderr=subprocess.STDOUT, stdout = subprocess.PIPE,
universal_newlines=True)
+ stop_recording_time(test_name)
script_output = proc.communicate(timeout=timeout)[0]
test_passed = not proc.wait()
@@ -729,6 +734,7 @@
sys.stdout.flush()
def print_analysis():
+ print_mutex.acquire()
if not verbose:
# Without --verbose, the testrunner erases passing test info. It
# does that by overriding the printed text with white spaces all across
@@ -762,6 +768,7 @@
print_text(COLOR_ERROR + '----------' + COLOR_NORMAL + '\n')
for failed_test in sorted([test_info[0] for test_info in failed_tests]):
print_text(('%s\n' % (failed_test)))
+ print_mutex.release()
def parse_test_name(test_name):
@@ -990,7 +997,33 @@
return test
+def start_recording_time(key):
+ """To begin recording time for the event associated with the key.
+ """
+ TIME_STATS[key] = -(time.time())
+
+def stop_recording_time(key):
+ """To stop timer for the event associated with the key.
+ """
+ TIME_STATS[key] = time.time() + TIME_STATS[key]
+
+def print_time_info():
+ """Print time information for different invocation.
+ """
+ print_mutex.acquire()
+ print_text('\nTIME INFO\n')
+ for key in TIME_STATS:
+ # Handle unfinised jobs.
+ if TIME_STATS[key] < 0:
+ TIME_STATS[key] = time.time() + TIME_STATS[key]
+
+ info_list = sorted(TIME_STATS.items(), key=operator.itemgetter(1), reverse=True)
+ for time_info_tuple in info_list:
+ print_text('%s : %.2f sec\n' % (time_info_tuple[0], time_info_tuple[1]))
+ print_mutex.release()
+
def main():
+ start_time = time.time()
gather_test_info()
user_requested_test = parse_option()
setup_test_env()
@@ -1006,8 +1039,10 @@
build_command += ' ' + build_targets
# Add 'dist' to avoid Jack issues b/36169180.
build_command += ' dist'
+ start_recording_time(build_command)
if subprocess.call(build_command.split()):
sys.exit(1)
+ stop_recording_time(build_command)
if user_requested_test:
test_runner_thread = threading.Thread(target=run_tests, args=(user_requested_test,))
else:
@@ -1016,6 +1051,13 @@
try:
test_runner_thread.start()
while threading.active_count() > 1:
+ if (time.time() - start_time > global_timeout):
+ # to ensure that the run ends before the go/ab bots
+ # time out the invocation.
+ print_text("FAILED: timeout reached")
+ print_time_info()
+ print_analysis()
+ sys.exit(1)
time.sleep(0.1)
print_analysis()
except Exception as e:
