| # |
| # Copyright (C) 2017 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| import base64 |
| import getpass |
| import logging |
| import os |
| import socket |
| import time |
| |
| from vts.proto import VtsReportMessage_pb2 as ReportMsg |
| from vts.runners.host import keys |
| from vts.utils.python.web import dashboard_rest_client |
| from vts.utils.python.web import feature_utils |
| |
| _PROFILING_POINTS = "profiling_points" |
| |
| |
| class WebFeature(feature_utils.Feature): |
| """Feature object for web functionality. |
| |
| Attributes: |
| enabled: boolean, True if web feature is enabled, False otherwise |
| report_msg: TestReportMessage, Proto summarizing the test run |
| current_test_report_msg: TestCaseReportMessage, Proto summarizing the current test case |
| rest_client: DashboardRestClient, client to which data will be posted |
| """ |
| |
| _TOGGLE_PARAM = keys.ConfigKeys.IKEY_ENABLE_WEB |
| _REQUIRED_PARAMS = [ |
| keys.ConfigKeys.IKEY_DASHBOARD_POST_COMMAND, |
| keys.ConfigKeys.IKEY_SERVICE_JSON_PATH, |
| keys.ConfigKeys.KEY_TESTBED_NAME, keys.ConfigKeys.IKEY_BUILD, |
| keys.ConfigKeys.IKEY_ANDROID_DEVICE, keys.ConfigKeys.IKEY_ABI_NAME, |
| keys.ConfigKeys.IKEY_ABI_BITNESS |
| ] |
| _OPTIONAL_PARAMS = [ |
| keys.ConfigKeys.RUN_AS_VTS_SELFTEST, |
| keys.ConfigKeys.IKEY_ENABLE_PROFILING, |
| ] |
| |
| def __init__(self, user_params): |
| """Initializes the web feature. |
| |
| Parses the arguments and initializes the web functionality. |
| |
| Args: |
| user_params: A dictionary from parameter name (String) to parameter value. |
| """ |
| self.ParseParameters( |
| toggle_param_name=self._TOGGLE_PARAM, |
| required_param_names=self._REQUIRED_PARAMS, |
| optional_param_names=self._OPTIONAL_PARAMS, |
| user_params=user_params) |
| if not self.enabled: |
| return |
| |
| # Initialize the dashboard client |
| post_cmd = getattr(self, keys.ConfigKeys.IKEY_DASHBOARD_POST_COMMAND) |
| service_json_path = str( |
| getattr(self, keys.ConfigKeys.IKEY_SERVICE_JSON_PATH)) |
| try: |
| self.rest_client = dashboard_rest_client.DashboardRestClient( |
| post_cmd, service_json_path) |
| if not self.rest_client.Initialize(): |
| self.enabled = False |
| except Exception as e: |
| logging.exception("Failed to create REST client.") |
| self.enabled = False |
| |
| self.report_msg = ReportMsg.TestReportMessage() |
| self.report_msg.test = str( |
| getattr(self, keys.ConfigKeys.KEY_TESTBED_NAME)) |
| |
| if getattr(self, keys.ConfigKeys.IKEY_ENABLE_PROFILING, False): |
| self.report_msg.test += "Profiling" |
| |
| self.report_msg.test_type = ReportMsg.VTS_HOST_DRIVEN_STRUCTURAL |
| self.report_msg.start_timestamp = feature_utils.GetTimestamp() |
| self.report_msg.host_info.hostname = socket.gethostname() |
| |
| android_devices = getattr(self, keys.ConfigKeys.IKEY_ANDROID_DEVICE, |
| None) |
| if not android_devices or not isinstance(android_devices, list): |
| logging.warn("android device information not available") |
| return |
| |
| for device_spec in android_devices: |
| dev_info = self.report_msg.device_info.add() |
| for elem in [ |
| keys.ConfigKeys.IKEY_PRODUCT_TYPE, |
| keys.ConfigKeys.IKEY_PRODUCT_VARIANT, |
| keys.ConfigKeys.IKEY_BUILD_FLAVOR, |
| keys.ConfigKeys.IKEY_BUILD_ID, keys.ConfigKeys.IKEY_BRANCH, |
| keys.ConfigKeys.IKEY_BUILD_ALIAS, |
| keys.ConfigKeys.IKEY_API_LEVEL, keys.ConfigKeys.IKEY_SERIAL |
| ]: |
| if elem in device_spec: |
| setattr(dev_info, elem, str(device_spec[elem])) |
| # TODO: get abi information differently for multi-device support. |
| setattr(dev_info, keys.ConfigKeys.IKEY_ABI_NAME, |
| str(getattr(self, keys.ConfigKeys.IKEY_ABI_NAME))) |
| setattr(dev_info, keys.ConfigKeys.IKEY_ABI_BITNESS, |
| str(getattr(self, keys.ConfigKeys.IKEY_ABI_BITNESS))) |
| |
| def SetTestResult(self, result=None): |
| """Set the current test case result to the provided result. |
| |
| If None is provided as a result, the current test report will be cleared, which results |
| in a silent skip. |
| |
| Requires the feature to be enabled; no-op otherwise. |
| |
| Args: |
| result: ReportMsg.TestCaseResult, the result of the current test or None. |
| """ |
| if not self.enabled: |
| return |
| |
| if not result: |
| self.report_msg.test_case.remove(self.current_test_report_msg) |
| self.current_test_report_msg = None |
| else: |
| self.current_test_report_msg.test_result = result |
| |
| def AddTestReport(self, test_name): |
| """Creates a report for the specified test. |
| |
| Requires the feature to be enabled; no-op otherwise. |
| |
| Args: |
| test_name: String, the name of the test |
| """ |
| if not self.enabled: |
| return |
| self.current_test_report_msg = self.report_msg.test_case.add() |
| self.current_test_report_msg.name = test_name |
| self.current_test_report_msg.start_timestamp = feature_utils.GetTimestamp( |
| ) |
| |
| def AddApiCoverageReport(self, api_coverage_data_vec, isGlobal=True): |
| """Adds an API coverage report to the VtsReportMessage. |
| |
| Translate each element in the give coverage data vector into a |
| ApiCoverageReportMessage within the report message. |
| |
| Args: |
| api_coverage_data_vec: list of VTSApiCoverageData which contains |
| the metadata (e.g. package_name, version) |
| and the total/covered api names. |
| isGlobal: boolean, True if the coverage data is for the entire test, |
| False if only for the current test case. |
| """ |
| |
| if not self.enabled: |
| return |
| |
| if isGlobal: |
| report = self.report_msg |
| else: |
| report = self.current_test_report_msg |
| |
| for api_coverage_data in api_coverage_data_vec: |
| api_coverage = report.api_coverage.add() |
| api_coverage.hal_interface.hal_package_name = api_coverage_data.package_name |
| api_coverage.hal_interface.hal_version_major = int( |
| api_coverage_data.version_major) |
| api_coverage.hal_interface.hal_version_minor = int( |
| api_coverage_data.version_minor) |
| api_coverage.hal_interface.hal_interface_name = api_coverage_data.interface_name |
| api_coverage.hal_api.extend(api_coverage_data.total_apis) |
| api_coverage.covered_hal_api.extend(api_coverage_data.covered_apis) |
| |
| def AddCoverageReport(self, |
| coverage_vec, |
| src_file_path, |
| git_project_name, |
| git_project_path, |
| revision, |
| covered_count, |
| line_count, |
| isGlobal=True): |
| """Adds a coverage report to the VtsReportMessage. |
| |
| Processes the source information, git project information, and processed |
| coverage information and stores it into a CoverageReportMessage within the |
| report message. |
| |
| Args: |
| coverage_vec: list, list of coverage counts (int) for each line |
| src_file_path: the path to the original source file |
| git_project_name: the name of the git project containing the source |
| git_project_path: the path from the root to the git project |
| revision: the commit hash identifying the source code that was used to |
| build a device image |
| covered_count: int, number of lines covered |
| line_count: int, total number of lines |
| isGlobal: boolean, True if the coverage data is for the entire test, False if only for |
| the current test case. |
| """ |
| if not self.enabled: |
| return |
| |
| if isGlobal: |
| report = self.report_msg |
| else: |
| report = self.current_test_report_msg |
| |
| coverage = report.coverage.add() |
| coverage.total_line_count = line_count |
| coverage.covered_line_count = covered_count |
| coverage.line_coverage_vector.extend(coverage_vec) |
| |
| src_file_path = os.path.relpath(src_file_path, git_project_path) |
| coverage.file_path = src_file_path |
| coverage.revision = revision |
| coverage.project_name = git_project_name |
| |
| def AddProfilingDataTimestamp( |
| self, |
| name, |
| start_timestamp, |
| end_timestamp, |
| x_axis_label="Latency (nano secs)", |
| y_axis_label="Frequency", |
| regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): |
| """Adds the timestamp profiling data to the web DB. |
| |
| Requires the feature to be enabled; no-op otherwise. |
| |
| Args: |
| name: string, profiling point name. |
| start_timestamp: long, nanoseconds start time. |
| end_timestamp: long, nanoseconds end time. |
| x-axis_label: string, the x-axis label title for a graph plot. |
| y-axis_label: string, the y-axis label title for a graph plot. |
| regression_mode: specifies the direction of change which indicates |
| performance regression. |
| """ |
| if not self.enabled: |
| return |
| |
| if not hasattr(self, _PROFILING_POINTS): |
| setattr(self, _PROFILING_POINTS, set()) |
| |
| if name in getattr(self, _PROFILING_POINTS): |
| logging.error("profiling point %s is already active.", name) |
| return |
| |
| getattr(self, _PROFILING_POINTS).add(name) |
| profiling_msg = self.report_msg.profiling.add() |
| profiling_msg.name = name |
| profiling_msg.type = ReportMsg.VTS_PROFILING_TYPE_TIMESTAMP |
| profiling_msg.regression_mode = regression_mode |
| profiling_msg.start_timestamp = start_timestamp |
| profiling_msg.end_timestamp = end_timestamp |
| profiling_msg.x_axis_label = x_axis_label |
| profiling_msg.y_axis_label = y_axis_label |
| |
| def AddProfilingDataVector( |
| self, |
| name, |
| labels, |
| values, |
| data_type, |
| options=[], |
| x_axis_label="x-axis", |
| y_axis_label="y-axis", |
| regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): |
| """Adds the vector profiling data in order to upload to the web DB. |
| |
| Requires the feature to be enabled; no-op otherwise. |
| |
| Args: |
| name: string, profiling point name. |
| labels: a list or set of labels. |
| values: a list or set of values where each value is an integer. |
| data_type: profiling data type. |
| options: a set of options. |
| x-axis_label: string, the x-axis label title for a graph plot. |
| y-axis_label: string, the y-axis label title for a graph plot. |
| regression_mode: specifies the direction of change which indicates |
| performance regression. |
| """ |
| if not self.enabled: |
| return |
| |
| if not hasattr(self, _PROFILING_POINTS): |
| setattr(self, _PROFILING_POINTS, set()) |
| |
| if name in getattr(self, _PROFILING_POINTS): |
| logging.error("profiling point %s is already active.", name) |
| return |
| |
| getattr(self, _PROFILING_POINTS).add(name) |
| profiling_msg = self.report_msg.profiling.add() |
| profiling_msg.name = name |
| profiling_msg.type = data_type |
| profiling_msg.regression_mode = regression_mode |
| if labels: |
| profiling_msg.label.extend(labels) |
| profiling_msg.value.extend(values) |
| profiling_msg.x_axis_label = x_axis_label |
| profiling_msg.y_axis_label = y_axis_label |
| profiling_msg.options.extend(options) |
| |
| def AddProfilingDataLabeledVector( |
| self, |
| name, |
| labels, |
| values, |
| options=[], |
| x_axis_label="x-axis", |
| y_axis_label="y-axis", |
| regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): |
| """Adds the labeled vector profiling data in order to upload to the web DB. |
| |
| Requires the feature to be enabled; no-op otherwise. |
| |
| Args: |
| name: string, profiling point name. |
| labels: a list or set of labels. |
| values: a list or set of values where each value is an integer. |
| options: a set of options. |
| x-axis_label: string, the x-axis label title for a graph plot. |
| y-axis_label: string, the y-axis label title for a graph plot. |
| regression_mode: specifies the direction of change which indicates |
| performance regression. |
| """ |
| self.AddProfilingDataVector( |
| name, labels, values, ReportMsg.VTS_PROFILING_TYPE_LABELED_VECTOR, |
| options, x_axis_label, y_axis_label, regression_mode) |
| |
| def AddProfilingDataUnlabeledVector( |
| self, |
| name, |
| values, |
| options=[], |
| x_axis_label="x-axis", |
| y_axis_label="y-axis", |
| regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING): |
| """Adds the unlabeled vector profiling data in order to upload to the web DB. |
| |
| Requires the feature to be enabled; no-op otherwise. |
| |
| Args: |
| name: string, profiling point name. |
| values: a list or set of values where each value is an integer. |
| options: a set of options. |
| x-axis_label: string, the x-axis label title for a graph plot. |
| y-axis_label: string, the y-axis label title for a graph plot. |
| regression_mode: specifies the direction of change which indicates |
| performance regression. |
| """ |
| self.AddProfilingDataVector( |
| name, None, values, ReportMsg.VTS_PROFILING_TYPE_UNLABELED_VECTOR, |
| options, x_axis_label, y_axis_label, regression_mode) |
| |
| def AddSystraceUrl(self, url): |
| """Creates a systrace report message with a systrace URL. |
| |
| Adds a systrace report to the current test case report and supplies the |
| url to the systrace report. |
| |
| Requires the feature to be enabled; no-op otherwise. |
| |
| Args: |
| url: String, the url of the systrace report. |
| """ |
| if not self.enabled: |
| return |
| systrace_msg = self.current_test_report_msg.systrace.add() |
| systrace_msg.url.append(url) |
| |
| def AddLogUrls(self, urls): |
| """Creates a log message with log file URLs. |
| |
| Adds a log message to the current test module report and supplies the |
| url to the log files. |
| |
| Requires the feature to be enabled; no-op otherwise. |
| |
| Args: |
| urls: list of string, the URLs of the logs. |
| """ |
| if not self.enabled or urls is None: |
| return |
| |
| for url in urls: |
| for log_msg in self.report_msg.log: |
| if log_msg.url == url: |
| continue |
| |
| log_msg = self.report_msg.log.add() |
| log_msg.url = url |
| log_msg.name = os.path.basename(url) |
| |
| def GetTestModuleKeys(self): |
| """Returns the test module name and start timestamp. |
| |
| Those two values can be used to find the corresponding entry |
| in a used nosql database without having to lock all the data |
| (which is infesiable) thus are essential for strong consistency. |
| """ |
| return self.report_msg.test, self.report_msg.start_timestamp |
| |
| def GenerateReportMessage(self, requested, executed): |
| """Uploads the result to the web service. |
| |
| Requires the feature to be enabled; no-op otherwise. |
| |
| Args: |
| requested: list, A list of test case records requested to run |
| executed: list, A list of test case records that were executed |
| |
| Returns: |
| binary string, serialized report message. |
| None if web is not enabled. |
| """ |
| if not self.enabled: |
| return None |
| |
| for test in requested[len(executed):]: |
| msg = self.report_msg.test_case.add() |
| msg.name = test.test_name |
| msg.start_timestamp = feature_utils.GetTimestamp() |
| msg.end_timestamp = msg.start_timestamp |
| msg.test_result = ReportMsg.TEST_CASE_RESULT_FAIL |
| |
| self.report_msg.end_timestamp = feature_utils.GetTimestamp() |
| |
| build = getattr(self, keys.ConfigKeys.IKEY_BUILD) |
| if keys.ConfigKeys.IKEY_BUILD_ID in build: |
| build_id = str(build[keys.ConfigKeys.IKEY_BUILD_ID]) |
| self.report_msg.build_info.id = build_id |
| |
| logging.debug("_tearDownClass hook: start (username: %s)", |
| getpass.getuser()) |
| |
| if len(self.report_msg.test_case) == 0: |
| logging.warn("_tearDownClass hook: skip uploading (no test case)") |
| return '' |
| |
| post_msg = ReportMsg.DashboardPostMessage() |
| post_msg.test_report.extend([self.report_msg]) |
| |
| self.rest_client.AddAuthToken(post_msg) |
| |
| message_b = base64.b64encode(post_msg.SerializeToString()) |
| |
| logging.debug('Result proto message generated. size: %s', |
| len(message_b)) |
| |
| logging.debug("_tearDownClass hook: status upload time stamp %s", |
| str(self.report_msg.start_timestamp)) |
| |
| return message_b |
