chromium/tools/run_webview_cts.py

#!/usr/bin/env python
#
# Copyright (C) 2012 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Executes WebView CTS tests and verifies results against known failures.
"""

import re
import signal
import subprocess
import sys

# Eventually this list will be empty!
# If you add or remove tests from this lists please update the CTS
# spreadsheet!
EXPECTED_FAILURES = set([
  'android.webkit.cts.WebViewClientTest#testOnScaleChanged',
  'android.webkit.cts.WebViewTest#testCapturePicture',
  # BUG=crbug.com/162967
  'android.webkit.cts.WebViewTest#testPageScroll',
  'android.webkit.cts.WebViewTest#testRequestChildRectangleOnScreen',
  'android.webkit.cts.WebViewTest#testScrollBarOverlay',
  'android.webkit.cts.WebViewTest#testSetInitialScale',
  'android.webkit.cts.WebViewTest#testSetScrollBarStyle',
  'android.webkit.cts.WebViewTest#testSetWebViewClient',
  'android.webkit.cts.WebViewTest#testZoom',
  # b/8763845
  'android.webkit.cts.WebViewTest#testFindAll',
  # b/8763944
  'android.webkit.cts.WebViewTest#testGetContentHeight',
  # See b/8231433 for Geolocation test failures.
  'android.webkit.cts.GeolocationTest#testSimpleGeolocationRequestAcceptAlways',
  'android.webkit.cts.GeolocationTest#testSimpleGeolocationRequestReject',
  'android.webkit.cts.GeolocationTest#testSimpleGeolocationRequestAcceptOnce',
  # b/9159785
  'android.webkit.cts.WebViewTest#testFindNext',
  # b/5006389
  'android.webkit.cts.WebViewTest#testFlingScroll',
  # b/9103603
  'android.webkit.cts.WebViewTest#testRequestImageRef',
  # b/9121594
  'android.webkit.cts.WebHistoryItemTest#testWebHistoryItem',
  # Don't forget to update the spreadsheet! :)
])

def main():
  proc = None

  # Send INT signal to test runner and exit gracefully so not to lose all
  # output information in a run.
  def handler(signum, frame):
    if proc:
      proc.send_signal(signum)
  signal.signal(signal.SIGINT, handler)

  proc = subprocess.Popen(
      ['cts-tradefed', 'run', 'singleCommand', 'cts', '-p', 'android.webkit', '--screenshot-on-failure'],
      stdout=subprocess.PIPE, stderr=subprocess.PIPE)

  (stdout, stderr) = proc.communicate();

  passes = set(re.findall(r'.*: (.*) PASS', stdout))
  failures = set(re.findall(r'.*: (.*) FAIL', stdout))
  test_results = '%d passes; %d failures' % (len(passes), len(failures))

  unexpected_passes = EXPECTED_FAILURES.difference(failures)
  if len(unexpected_passes) > 0:
    test_results += '\n' + 'UNEXPECTED PASSES (update expectations!):'
    for test in unexpected_passes:
      test_results += '\n' + '\t%s' % (test)

  unexpected_failures = failures.difference(EXPECTED_FAILURES)
  if len(unexpected_failures) > 0:
    test_results += '\n' + 'UNEXPECTED FAILURES (please fix!):'
    for test in unexpected_failures:
      test_results += '\n' + '\t%s' % (test)

  unexpected_failures_count = len(unexpected_failures)
  unexpected_passes_count = len(unexpected_passes)

  # on the buildbot this is most useful at the start
  print test_results

  print '\nstdout dump follows...'
  print stdout
  print '\n'

  # on the cmd line this is most useful at the end
  print test_results

  # Allow buildbot script to distinguish failures and possibly out of date
  # test expectations.
  if len(passes) + len(failures) < 100:
    print 'Ran less than 100 cts tests? Something must be wrong'
    return 2
  elif unexpected_failures_count > 0:
    return 1
  elif unexpected_passes_count >= 5:
    print ('More than 5 new passes? Either you''re running webview classic, or '
           'it really is time to fix failure expectations.')
    return 2
  elif unexpected_passes_count > 0:
    return 3  # STEP_WARNINGS
  else:
    return 0


if __name__ == '__main__':
  sys.exit(main())