| #!/usr/bin/env python3 |
| # -*- coding: utf-8 -*- |
| # |
| # Copyright 2017 The ChromiumOS Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Script for running llvm validation tests on ChromeOS. |
| |
| This script launches a buildbot to build ChromeOS with the llvm on |
| a particular board; then it finds and downloads the trybot image and the |
| corresponding official image, and runs test for correctness. |
| It then generates a report, emails it to the c-compiler-chrome, as |
| well as copying the result into a directory. |
| """ |
| |
| # Script to test different toolchains against ChromeOS benchmarks. |
| |
| |
| import argparse |
| import datetime |
| import os |
| import sys |
| import time |
| |
| from cros_utils import buildbot_utils |
| from cros_utils import command_executer |
| from cros_utils import logger |
| |
| |
| CROSTC_ROOT = "/usr/local/google/crostc" |
| ROLE_ACCOUNT = "mobiletc-prebuild" |
| TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) |
| MAIL_PROGRAM = "~/var/bin/mail-detective" |
| VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, "validation_result") |
| START_DATE = datetime.date(2016, 1, 1) |
| TEST_PER_DAY = 4 |
| DATA_DIR = "/google/data/rw/users/mo/mobiletc-prebuild/waterfall-report-data/" |
| |
| # Information about Rotating Boards |
| # Board Arch Reference Platform Kernel |
| # Board Version |
| # ------------ ------- ------------ ------------- ------- |
| # atlas x86_64 poppy kabylake-y 4.4.* |
| # cave x86_64 glados skylake-y 3.18 |
| # coral x86_64 reef apollo lake 4.4.* |
| # cyan x86_64 strago braswell 3.18 |
| # elm aarch64 oak mediatek-8173 3.18 arm32 userspace |
| # eve x86_64 poppy kabylake-u/r 4.4.* |
| # gale armv7 3.18 |
| # grunt x86_64 grunt stoney ridge 4.14.* |
| # fizz-moblab x86_64 4.4 |
| # kevin aarch64 gru rockchip-3399 4.4.* arm32 userspace |
| # kevin64 aarch64 gru rockchip-3399 4.4.* arm64 userspace |
| # lakitu x86_64 4.4.* |
| # nyan_kitty armv7 nyan tegra 3.10.18 |
| # octopus x86_64 octopus GLK 4.14.* |
| # sentry x86_64 kunimitsu skylake-u 3.18 |
| # tidus x86_64 auron broadwell 3.14 |
| # veyron_mighty armv7 veyron-pinky rockchip-3288 3.14 |
| # whirlwind armv7 3.14 |
| # winky x86_64 rambi baytrail 4.4.* |
| |
| TEST_BOARD = [ |
| "atlas", |
| "cave", |
| "coral", |
| "cyan", |
| "elm", |
| # 'eve', tested by amd64-llvm-next-toolchain builder. |
| "gale", |
| "grunt", |
| "fizz-moblab", |
| # 'kevin', tested by arm64-llvm-next-toolchain builder. |
| "kevin64", |
| "lakitu", |
| "nyan_kitty", |
| "octopus", |
| "sentry", |
| "tidus", |
| # 'veyron_mighty', tested by arm-llvm-next-toolchain builder. |
| "whirlwind", |
| "winky", |
| ] |
| |
| |
| class ToolchainVerifier(object): |
| """Class for the toolchain verifier.""" |
| |
| def __init__(self, board, chromeos_root, weekday, patches, compiler): |
| self._board = board |
| self._chromeos_root = chromeos_root |
| self._base_dir = os.getcwd() |
| self._ce = command_executer.GetCommandExecuter() |
| self._l = logger.GetLogger() |
| self._compiler = compiler |
| self._build = "%s-%s-toolchain-tryjob" % (board, compiler) |
| self._patches = patches.split(",") if patches else [] |
| self._patches_string = "_".join(str(p) for p in self._patches) |
| |
| if not weekday: |
| self._weekday = time.strftime("%a") |
| else: |
| self._weekday = weekday |
| self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board) |
| |
| def DoAll(self): |
| """Main function inside ToolchainComparator class. |
| |
| Launch trybot, get image names, create crosperf experiment file, run |
| crosperf, and copy images into seven-day report directories. |
| """ |
| buildbucket_id, _ = buildbot_utils.GetTrybotImage( |
| self._chromeos_root, |
| self._build, |
| self._patches, |
| tryjob_flags=["--hwtest"], |
| asynchronous=True, |
| ) |
| |
| return buildbucket_id |
| |
| |
| def WriteRotatingReportsData(results_dict, date): |
| """Write data for waterfall report.""" |
| fname = "%d-%02d-%02d.builds" % (date.year, date.month, date.day) |
| filename = os.path.join(DATA_DIR, "rotating-builders", fname) |
| with open(filename, "w", encoding="utf-8") as out_file: |
| for board in results_dict.keys(): |
| buildbucket_id = results_dict[board] |
| out_file.write("%s,%s\n" % (buildbucket_id, board)) |
| |
| |
| def Main(argv): |
| """The main function.""" |
| |
| # Common initializations |
| command_executer.InitCommandExecuter() |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| "--chromeos_root", |
| dest="chromeos_root", |
| help="The chromeos root from which to run tests.", |
| ) |
| parser.add_argument( |
| "--weekday", |
| default="", |
| dest="weekday", |
| help="The day of the week for which to run tests.", |
| ) |
| parser.add_argument( |
| "--board", default="", dest="board", help="The board to test." |
| ) |
| parser.add_argument( |
| "--patch", |
| dest="patches", |
| default="", |
| help="The patches to use for the testing, " |
| "seprate the patch numbers with ',' " |
| "for more than one patches.", |
| ) |
| parser.add_argument( |
| "--compiler", |
| dest="compiler", |
| help="Which compiler (llvm, llvm-next or gcc) to use for " "testing.", |
| ) |
| |
| options = parser.parse_args(argv[1:]) |
| if not options.chromeos_root: |
| print("Please specify the ChromeOS root directory.") |
| return 1 |
| if not options.compiler: |
| print( |
| "Please specify which compiler to test (gcc, llvm, or llvm-next)." |
| ) |
| return 1 |
| |
| if options.board: |
| fv = ToolchainVerifier( |
| options.board, |
| options.chromeos_root, |
| options.weekday, |
| options.patches, |
| options.compiler, |
| ) |
| return fv.DoAll() |
| |
| today = datetime.date.today() |
| delta = today - START_DATE |
| days = delta.days |
| |
| start_board = (days * TEST_PER_DAY) % len(TEST_BOARD) |
| results_dict = dict() |
| for i in range(TEST_PER_DAY): |
| try: |
| board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)] |
| fv = ToolchainVerifier( |
| board, |
| options.chromeos_root, |
| options.weekday, |
| options.patches, |
| options.compiler, |
| ) |
| buildbucket_id = fv.DoAll() |
| if buildbucket_id: |
| results_dict[board] = buildbucket_id |
| except SystemExit: |
| logfile = os.path.join( |
| VALIDATION_RESULT_DIR, options.compiler, board |
| ) |
| with open(logfile, "w", encoding="utf-8") as f: |
| f.write("Verifier got an exception, please check the log.\n") |
| WriteRotatingReportsData(results_dict, today) |
| |
| |
| if __name__ == "__main__": |
| retval = Main(sys.argv) |
| sys.exit(retval) |