| #!/usr/bin/env python |
| # |
| # Copyright (C) 2016 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| """Runs the test suite over a set of devices.""" |
| from __future__ import print_function |
| |
| import argparse |
| import distutils.spawn |
| import os |
| import re |
| import shutil |
| import signal |
| import site |
| import subprocess |
| import sys |
| import tempfile |
| |
| import ndk.debug |
| |
| |
| THIS_DIR = os.path.realpath(os.path.dirname(__file__)) |
| |
| |
| class Device(object): |
| def __init__(self, serial, name, version, build_id, abis, is_emulator): |
| self.serial = serial |
| self.name = name |
| self.version = version |
| self.build_id = build_id |
| self.abis = abis |
| self.is_emulator = is_emulator |
| |
| def __str__(self): |
| return 'android-{} {} {} {}'.format( |
| self.version, self.name, self.serial, self.build_id) |
| |
| |
| class DeviceFleet(object): |
| def __init__(self): |
| self.devices = { |
| 10: { |
| 'armeabi': None, |
| 'armeabi-v7a': None, |
| }, |
| 16: { |
| 'armeabi': None, |
| 'armeabi-v7a': None, |
| 'x86': None, |
| }, |
| 23: { |
| 'armeabi': None, |
| 'armeabi-v7a': None, |
| 'arm64-v8a': None, |
| 'x86': None, |
| 'x86_64': None, |
| }, |
| } |
| |
| def add_device(self, device): |
| if device.version not in self.devices: |
| print('Ignoring device for unwanted API level: {}'.format(device)) |
| return |
| |
| same_version = self.devices[device.version] |
| for abi, current_device in same_version.iteritems(): |
| # This device can't fulfill this ABI. |
| if abi not in device.abis: |
| continue |
| |
| # Anything is better than nothing. |
| if current_device is None: |
| self.devices[device.version][abi] = device |
| continue |
| |
| # The emulator images have actually been changed over time, so the |
| # devices are more trustworthy. |
| if current_device.is_emulator and not device.is_emulator: |
| self.devices[device.version][abi] = device |
| |
| def get_device(self, version, abi): |
| return self.devices[version][abi] |
| |
| def get_missing(self): |
| missing = [] |
| for version, abis in self.devices.iteritems(): |
| for abi, device in abis.iteritems(): |
| if device is None: |
| missing.append('android-{} {}'.format(version, abi)) |
| return missing |
| |
| def get_versions(self): |
| return self.devices.keys() |
| |
| def get_abis(self, version): |
| return self.devices[version].keys() |
| |
| |
| def get_device_abis(properties): |
| # 64-bit devices list their ABIs differently than 32-bit devices. Check all |
| # the possible places for stashing ABI info and merge them. |
| abi_properties = [ |
| 'ro.product.cpu.abi', |
| 'ro.product.cpu.abi2', |
| 'ro.product.cpu.abilist', |
| ] |
| abis = set() |
| for abi_prop in abi_properties: |
| if abi_prop in properties: |
| abis.update(properties[abi_prop].split(',')) |
| |
| return sorted(list(abis)) |
| |
| |
| def get_device_details(serial): |
| import adb # pylint: disable=import-error |
| props = adb.get_device(serial).get_props() |
| name = props['ro.product.name'] |
| version = int(props['ro.build.version.sdk']) |
| supported_abis = get_device_abis(props) |
| build_id = props['ro.build.id'] |
| is_emulator = props.get('ro.build.characteristics') == 'emulator' |
| return Device(serial, name, version, build_id, supported_abis, is_emulator) |
| |
| |
| def find_devices(): |
| """Detects connected devices and returns a set for testing. |
| |
| We get a list of devices by scanning the output of `adb devices`. We want |
| to run the tests for the cross product of the following configurations: |
| |
| ABIs: {armeabi, armeabi-v7a, arm64-v8a, mips, mips64, x86, x86_64} |
| Platform versions: {android-10, android-16, android-21} |
| Toolchains: {clang, gcc} |
| |
| Note that not all ABIs are available for every platform version. There are |
| no 64-bit ABIs before android-21, and there were no MIPS ABIs for |
| android-10. |
| """ |
| if distutils.spawn.find_executable('adb') is None: |
| raise RuntimeError('Could not find adb.') |
| |
| # We could get the device name from `adb devices -l`, but we need to |
| # getprop to find other details anyway, and older devices don't report |
| # their names properly (nakasi on android-16, for example). |
| p = subprocess.Popen(['adb', 'devices'], stdout=subprocess.PIPE) |
| out, _ = p.communicate() |
| if p.returncode != 0: |
| raise RuntimeError('Failed to get list of devices from adb.') |
| |
| # The first line of `adb devices` just says "List of attached devices", so |
| # skip that. |
| fleet = DeviceFleet() |
| for line in out.split('\n')[1:]: |
| if not line.strip(): |
| continue |
| |
| serial, _ = re.split(r'\s+', line, maxsplit=1) |
| |
| if 'offline' in line: |
| print('Ignoring offline device: ' + serial) |
| continue |
| if 'unauthorized' in line: |
| print('Ignoring unauthorized device: ' + serial) |
| continue |
| |
| device = get_device_details(serial) |
| print('Found device {}'.format(device)) |
| fleet.add_device(device) |
| |
| return fleet |
| |
| |
| def parse_args(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| 'ndk', metavar='NDK', type=os.path.realpath, help='NDK to validate.') |
| parser.add_argument( |
| '--filter', help='Only run tests that match the given pattern.') |
| parser.add_argument( |
| '--log-dir', type=os.path.realpath, default='test-logs', |
| help='Directory to store test logs.') |
| |
| return parser.parse_args() |
| |
| |
| def get_aggregate_results(details): |
| tests = {} |
| for version, abis in details.iteritems(): |
| for abi, toolchains in abis.iteritems(): |
| for toolchain, results in toolchains.iteritems(): |
| for suite, test_results in results.iteritems(): |
| for test in test_results: |
| if test.failed(): |
| name = '.'.join([suite, test.test_name]) |
| if name not in tests: |
| tests[name] = [] |
| tests[name].append((version, abi, toolchain, test)) |
| return tests |
| |
| |
| def print_aggregate_details(details, use_color): |
| tests = get_aggregate_results(details) |
| for test_name, configs in tests.iteritems(): |
| # We might be printing a lot of crap here, so let's be obvious about |
| # where each test starts. |
| print('BEGIN TEST RESULT: ' + test_name) |
| print('=' * 80) |
| |
| for version, abi, toolchain, result in configs: |
| print('FAILED {} {} {}:'.format(toolchain, abi, version)) |
| print(result.to_string(colored=use_color)) |
| |
| |
| def main(): |
| if sys.platform != 'win32': |
| ndk.debug.register_debug_handler(signal.SIGUSR1) |
| ndk.debug.register_trace_handler(signal.SIGUSR2) |
| |
| args = parse_args() |
| |
| os.chdir(THIS_DIR) |
| |
| # We need to do this here rather than at the top because we load the module |
| # from a path that is given on the command line. We load it from the NDK |
| # given on the command line so this script can be run even without a full |
| # platform checkout. |
| site.addsitedir(os.path.join(args.ndk, 'python-packages')) |
| |
| ndk_build_path = os.path.join(args.ndk, 'ndk-build') |
| if os.name == 'nt': |
| ndk_build_path += '.cmd' |
| if not os.path.exists(ndk_build_path): |
| sys.exit(ndk_build_path + ' does not exist.') |
| |
| fleet = find_devices() |
| print('Test configuration:') |
| for version in sorted(fleet.get_versions()): |
| print('\tandroid-{}:'.format(version)) |
| for abi in sorted(fleet.get_abis(version)): |
| print('\t\t{}: {}'.format(abi, fleet.get_device(version, abi))) |
| missing_configs = fleet.get_missing() |
| if len(missing_configs): |
| print('Missing configurations: {}'.format(', '.join(missing_configs))) |
| |
| if os.path.exists(args.log_dir): |
| shutil.rmtree(args.log_dir) |
| os.makedirs(args.log_dir) |
| |
| use_color = sys.stdin.isatty() and os.name != 'nt' |
| out_dir = tempfile.mkdtemp() |
| try: |
| import tests.runners |
| good, details = tests.runners.run_for_fleet( |
| args.ndk, fleet, out_dir, args.log_dir, args.filter, use_color) |
| finally: |
| shutil.rmtree(out_dir) |
| |
| print_aggregate_details(details, use_color) |
| |
| sys.exit(not good) |
| |
| |
| if __name__ == '__main__': |
| main() |
