tools/compare_failed_tests/compare_failed_tests.py

#!/usr/bin/python
"""Compare failed tests in CTS/VTS test_result.xml.

Given two test_result.xml's (A and B), this script lists all failed tests in A,
and shows result of the same test in B.
"""

import argparse
import collections
import csv
import xml.etree.ElementTree as ET


PASS = 'pass'
FAIL = 'fail'
NO_DATA = 'no_data'

ATTRS_TO_SHOW = ['Result::Build.build_model',
                 'Result::Build.build_id',
                 'Result.suite_name',
                 'Result.suite_plan',
                 'Result.suite_build_number',
                 'Result.start_display',
                 'Result::Build.build_abis_32',
                 'Result::Build.build_abis_64',]


def parse_attrib_path(attrib_path):
  first_dot = attrib_path.index('.')
  tags = attrib_path[:first_dot].split('::')
  attr_name = attrib_path[first_dot+1:]
  return tags, attr_name


def get_test_info(root):
  """Get test info from test_result.xml."""

  test_info = collections.OrderedDict()

  for attrib_path in ATTRS_TO_SHOW:
    tags, attr_name = parse_attrib_path(attrib_path)
    node = root

    while True:
      tags = tags[1:]
      if tags:
        node = node.find(tags[0])
      else:
        break

    test_info[attr_name] = node.attrib[attr_name]

  return test_info


def print_test_infos(test_result_a, test_result_b):
  """Print test infomation of both results in table format."""

  info_a = test_result_a['info']
  info_b = test_result_b['info']

  max_key_len = max([len(k) for k in info_a])
  max_value_a_len = max([len(info_a[k]) for k in info_a])
  max_value_b_len = max([len(info_b[k]) for k in info_b])
  table_len = (max_key_len + 2 + max_value_a_len + 2 + max_value_b_len)

  line_format = '{:{}}  {:{}}  {}'

  print '=' * table_len

  for key in info_a:
    print line_format.format(key, max_key_len,
                             info_a[key], max_value_a_len,
                             info_b[key])

  print '=' * table_len
  print


def get_result(test_result, module_name, testcase_name, test_name):
  """Get result of specifc module, testcase and test name."""

  modules = test_result['modules']
  if module_name not in modules:
    return NO_DATA

  testcases = modules[module_name]
  if testcase_name not in testcases:
    return NO_DATA

  tests = testcases[testcase_name]
  if test_name not in tests:
    return NO_DATA

  return ', '.join([x + ': ' + y for x, y in tests[test_name].items()])


def read_test_result_xml(test_result_path):
  """Given the path to a test_result.xml, read that into a ordered dict."""

  tree = ET.parse(test_result_path)
  root = tree.getroot()

  test_result = collections.OrderedDict()
  test_result['info'] = get_test_info(root)

  modules = collections.OrderedDict()
  test_result['modules'] = modules

  for module in root.iter('Module'):
    abi = module.attrib['abi']

    module_name = module.attrib['name']

    if not module_name in modules:
      modules[module_name] = collections.OrderedDict()

    testcases = modules[module_name]

    for testcase in module.iter('TestCase'):
      testcase_name = testcase.attrib['name']

      if not testcase_name in testcases:
        testcases[testcase_name] = collections.OrderedDict()

      tests = testcases[testcase_name]

      for test in testcase.iter('Test'):
        test_name = test.attrib['name']

        if not test_name in tests:
          tests[test_name] = collections.OrderedDict()

        if abi in tests[test_name]:
          print '[WARNING] duplicated test:', test_name

        tests[test_name][abi] = test.attrib['result']

  return test_result


def compare_failed_tests(test_result_a, test_result_b, csvfile):
  """Do the comparison.

  Given two test result dicts (A and B), list all failed test in A and display
  result of the same test in B.

  Args:
    test_result_a: the dict returned from read_test_result(test_result_a.xml)
    test_result_b: the dict returned from read_test_result(test_result_b.xml)
    csvfile: a opened file

  Returns:
    string: diff report, summary
  """

  writer = csv.writer(csvfile)
  writer.writerow(['module', 'testcase', 'test', 'result in B'])

  summary = ''

  modules = test_result_a['modules']

  for module_name, testcases in modules.iteritems():
    module_sub_summary = ''

    for testcase_name, tests in testcases.iteritems():
      testcase_sub_summary = ''

      for test_name, result in tests.iteritems():
        if FAIL in result.values():
          result_b = get_result(
              test_result_b, module_name, testcase_name, test_name)

          testcase_sub_summary += '    ' + test_name + ': ' + result_b + '\n'
          writer.writerow([module_name, testcase_name, test_name, result_b])

      if testcase_sub_summary:
        module_sub_summary = '  ' + testcase_name + '\n' + testcase_sub_summary

    if module_sub_summary:
      summary += module_name + '\n' + module_sub_summary + '\n'

  return summary


def main():
  parser = argparse.ArgumentParser()

  parser.add_argument('test_result_a', help='path to first test_result.xml')
  parser.add_argument('test_result_b', help='path to second test_result.xml')
  parser.add_argument('--csv', default='diff.csv', help='path to csv output')

  args = parser.parse_args()

  test_result_a = read_test_result_xml(args.test_result_a)
  test_result_b = read_test_result_xml(args.test_result_b)

  print_test_infos(test_result_a, test_result_b)

  with open(args.csv, 'w') as csvfile:
    summary = compare_failed_tests(test_result_a, test_result_b, csvfile)

    print summary


if __name__ == '__main__':
  main()