tests/tests/webkitsecurity/generate/postprocess.py

#! /usr/bin/env python

"""
webkit_security_postprocessor.py

This script postprocesses the result of a cts-tradefed run of webkit security tests to match the
failing tests to the appropriate CL in upstream WebKit.
"""

import os
import commands
import multiprocessing
from xml.etree import ElementTree

class Test:

  java_name = ""
  webkit_name = ""
  local_path = ""
  cls = None
  test_case = ""

  def __init__(self, java_name):
    self.java_name = java_name
    self.cls = []

  def __str__(self):
    data = self.java_name + '\n'
    underline = "=" * (len(data) - 1) + "\n"
    data += underline + '\n'
    data += "\t" + "to reproduce" + "\n"
    data += "\t" + "------------" + "\n"
    data += "\t" + "cts-tradefed run cts --class android.webkitsecurity.cts." + self.java_name + "\n"
    data += "\n"
    data += "\t" + "test case" + "\n"
    data += "\t" + "---------" + "\n"
    for line in self.test_case.splitlines():
      data += "\t" + line + "\n"
    data += "\n"
    data += "\t" + "revisions" + "\n"
    data += "\t" + "---------" + "\n"
    for cl in self.cls:
      name = "[" + str(cl) + "]"
      link = "http://trac.webkit.org/changeset/" + str(cl)[1:]
      data += "\t" + name + "(" + link + ")" + '\n'
    return data

# get the location of the xml output file
test_results_directory = "/usr/local/google/android/out/host/linux-x86/cts/android-cts/repository/results"


# get the location of the android webkitsecurity tests
source_dir = "/usr/local/google/android/"
android_tests_dir = source_dir + "cts/tests/tests/webkitsecurity/src/android/webkitsecurity/cts/"

# get the location of the webkit layout tests
webkit_tests_dir = "/usr/local/google/webkit/LayoutTests"

def get_test_results():
  # look in the test results directory
  test_results = []
  for root, dirs, fnames in os.walk(test_results_directory):
    for fname in fnames:
      if fname == 'testResult.xml':
        full_path = os.path.abspath(os.path.join(root, fname))
        test_results.append(full_path)

  # special cases
  if not test_results:
    raise Exception("No results found")
  elif len(test_results) == 1:
    return test_results[0]
  else:
    # find the timestamp of each file
    newest = None
    newest_fname = None
    for fname in test_results:
      timestamp_start = fname.index('20')
      timestamp_end = fname.index('/', start)
      timestamp = time.strptime(fname[timestamp_start:timestamp_end], "%Y.%m.%d_%H.%M.%S")
      if newest == None or timestamp > newest:
        newest = timestamp
        newest_fname = fname
    return newest_fname

def get_all_java_tests():
  element_tree = ElementTree.parse(get_test_results())
  for elem in element_tree.getiterator():
    if elem.tag == 'TestCase':
      java_name = elem.attrib['name']
      if java_name.startswith('Webkit') and java_name.endswith('Test'):
        for subelem in elem:
          if subelem.tag == 'Test':
            if subelem.attrib['result'] == 'fail':
              yield Test(java_name)

def extract_webkit_name_from_test(test):
  sig = "    private static final String TEST_PATH = \""
  for line in test:
    if line.startswith(sig):
      return line[len(sig):].strip()[:-2]

def get_all_web_tests(java_tests, android_test_dir):
  # for each java test
  for test in java_tests:
    test_name = os.path.join(android_test_dir, test.java_name) + '.java'
    with open(test_name) as f:
      # get the webkit test name
      webkit_test_name = extract_webkit_name_from_test(f)
      # postprocess it
      test.webkit_name = webkit_test_name.split('/')[-1]
      yield test

def find_local_files(tests, webkit_tests_dir):
  # convert the test names to a dictionary
  tests = dict((n.webkit_name, n) for n in tests)
  # for each test under LayoutTests/
  for root, dirs, files in os.walk(webkit_tests_dir):
    if '.svn' in root:
      continue
    for fname in files:
      if fname in tests:
        tests[fname].local_path = os.path.abspath(os.path.join(root, fname))
        with open(tests[fname].local_path) as f:
          tests[fname].test_case = f.read()
  # return the tests
  return tests.values()

def get_test_cls(test):
  cwd = os.getcwd()
  os.chdir(webkit_tests_dir)
  while True:
    cmd = "svn log -q %s" % test.local_path
    status, output = commands.getstatusoutput(cmd)
    if not status:
      break
  # parse the results, saving only the commit numbers
  for line in output.splitlines():
    if line.startswith('r'):
      cl = line.split()[0].strip()
      test.cls.append(cl)
  os.chdir(cwd)
  return test

def get_all_test_cls(tests):
  pool = multiprocessing.Pool(processes=12)
  return pool.map(get_test_cls, tests)

def output(tests, bugreport):
  for test in tests:
    print str(test)
  print
  print
  print bugreport

def get_crashes():
  status, output = commands.getstatusoutput("adb bugreport")
  # TODO reintroduce crash parsing
  # - this was removed due to questions about symbol resolution, but could be reintroduced if
  # - this script as a whole knew anything about branches.
  # - Alternatively, if this would fail meaningfully it could also be reintroduced
  return output

def decode_crashes(logcat):
  with open('/tmp/crashdump.txt', 'w') as f:
    f.write(logcat)
  cwd = os.getcwd()
  os.chdir('/usr/local/google/android') # TODO add branch setting
  status, output = commands.getstatusoutput("./vendor/google/tools/stack /tmp/crashdump.txt")
  return output


def post_bug(report):
  # TODO add auto bugging
  pass

if __name__ == "__main__":
  java_tests = list(get_all_java_tests())
  webkit_tests = list(get_all_web_tests(java_tests, android_tests_dir))
  local_tests = find_local_files(webkit_tests, webkit_tests_dir)
  tests = get_all_test_cls(local_tests)
  crashes = get_crashes()
  traces = decode_crashes(crashes)
  output(tests, traces)