[autotest] Report test result sizes to metrics
There are two reporting source:
1. Test jobs report result size information in tko/parse.
2. Special tasks report result size information at the end of autoserv process.
BUG=chromium:716218
TEST=unittest, local run test
Change-Id: I2410f3a3a5a16c1673446633e15c038a5e4ef81f
Reviewed-on: https://chromium-review.googlesource.com/571859
Commit-Ready: Dan Shi <dshi@google.com>
Tested-by: Dan Shi <dshi@google.com>
Reviewed-by: Dan Shi <dshi@google.com>
diff --git a/client/bin/result_tools/utils_lib.py b/client/bin/result_tools/utils_lib.py
index 4322e79..f9b33a7 100644
--- a/client/bin/result_tools/utils_lib.py
+++ b/client/bin/result_tools/utils_lib.py
@@ -4,6 +4,9 @@
"""Shared constants and methods for result utilities."""
+import collections
+
+
# Following are key names for directory summaries. The keys are started with /
# so it can be differentiated with a valid file name. The short keys are
# designed for smaller file size of the directory summary.
@@ -18,4 +21,61 @@
DIRS = '/D'
# Default root directory name. To allow summaries to be merged effectively, all
# summaries are collected with root directory of ''
-ROOT_DIR = ''
\ No newline at end of file
+ROOT_DIR = ''
+
+# Information of test result sizes to be stored in tko_job_keyvals.
+# The total size (in kB) of test results that generated during the test,
+# including:
+# * server side test logs and result files.
+# * client side test logs, sysinfo, system logs and crash dumps.
+# Note that a test can collect the same test result files from DUT multiple
+# times during the test, before and after each iteration/test. So the value of
+# client_result_collected_KB could be larger than the value of
+# result_uploaded_KB, which is the size of result directory on the server side,
+# even if the test result throttling is not applied.
+#
+# Attributes of the named tuple includes:
+# client_result_collected_KB: The total size (in KB) of test results collected
+# from test device.
+# original_result_total_KB: The original size (in KB) of test results before
+# being trimmed.
+# result_uploaded_KB: The total size (in KB) of test results to be uploaded by
+# gs_offloader.
+# result_throttled: Flag to indicate if test results collection is throttled.
+ResultSizeInfo = collections.namedtuple(
+ 'ResultSizeInfo',
+ ['client_result_collected_KB',
+ 'original_result_total_KB',
+ 'result_uploaded_KB',
+ 'result_throttled'])
+
+
+def get_result_size_info(client_collected_bytes, summary):
+ """Get the result size information.
+
+ @param client_collected_bytes: Size in bytes of results collected from the
+ test device.
+ @param summary: A dictionary of directory summary.
+ @return: A namedtuple of result size informations, including:
+ client_result_collected_KB: The total size (in KB) of test results
+ collected from test device.
+ original_result_total_KB: The original size (in KB) of test results
+ before being trimmed.
+ result_uploaded_KB: The total size (in KB) of test results to be
+ uploaded.
+ result_throttled: True if test results collection is throttled.
+ """
+ root_entry = summary[ROOT_DIR]
+ client_result_collected_KB= client_collected_bytes / 1024
+ original_result_total_KB = root_entry[ORIGINAL_SIZE_BYTES] / 1024
+ result_uploaded_KB = root_entry[TRIMMED_SIZE_BYTES] / 1024
+ # Test results are considered to be throttled if the total size of
+ # results collected is different from the total size of trimmed results
+ # from the client side.
+ result_throttled = (
+ root_entry[ORIGINAL_SIZE_BYTES] != root_entry[TRIMMED_SIZE_BYTES])
+
+ return ResultSizeInfo(client_result_collected_KB=client_result_collected_KB,
+ original_result_total_KB=original_result_total_KB,
+ result_uploaded_KB=result_uploaded_KB,
+ result_throttled=result_throttled)
\ No newline at end of file
diff --git a/server/autoserv b/server/autoserv
index 90b899c..a6c4422 100755
--- a/server/autoserv
+++ b/server/autoserv
@@ -26,6 +26,7 @@
from autotest_lib.client.common_lib import global_config
from autotest_lib.client.common_lib import utils
from autotest_lib.client.common_lib.cros.graphite import autotest_es
+from autotest_lib.server import site_utils
try:
from chromite.lib import metrics
@@ -558,6 +559,10 @@
finally:
job.close()
+ # Special task doesn't run parse, so result summary needs to be
+ # built here.
+ if results and (repair or verify or reset or cleanup or provision):
+ site_utils.collect_result_sizes(results)
except:
exit_code = 1
traceback.print_exc()
diff --git a/server/site_utils.py b/server/site_utils.py
index 4e8626f..d57a690 100644
--- a/server/site_utils.py
+++ b/server/site_utils.py
@@ -12,11 +12,16 @@
import random
import re
import time
+import traceback
import urllib2
import common
+from autotest_lib.client.bin.result_tools import utils as result_utils
+from autotest_lib.client.bin.result_tools import utils_lib as result_utils_lib
+from autotest_lib.client.bin.result_tools import view as result_view
from autotest_lib.client.common_lib import utils
from autotest_lib.client.common_lib import error
+from autotest_lib.client.common_lib import file_utils
from autotest_lib.client.common_lib import global_config
from autotest_lib.client.common_lib import host_queue_entry_states
from autotest_lib.client.common_lib import host_states
@@ -24,6 +29,11 @@
from autotest_lib.server.cros.dynamic_suite import constants
from autotest_lib.server.cros.dynamic_suite import job_status
+try:
+ from chromite.lib import metrics
+except ImportError:
+ metrics = utils.metrics_mock
+
CONFIG = global_config.global_config
@@ -53,6 +63,8 @@
'gm4g_sprout': 'seed_l8150',
'bat': 'bat_land'
}
+# Prefix for the metrics name for result size information.
+RESULT_METRICS_PREFIX = 'chromeos/autotest/result_collection/'
class TestLabException(Exception):
"""Exception raised when the Test Lab blocks a test or suite."""
@@ -807,7 +819,7 @@
@param duts: List of duts to check for idle state.
@param afe: afe instance.
- @param max_wait: Max wait time in seconds.
+ @param max_wait: Max wait time in seconds to wait for duts to be idle.
@returns Boolean True if all hosts are idle or False if any hosts did not
go idle within max_wait.
@@ -847,7 +859,7 @@
@param duts: List of duts to lock.
@param afe: afe instance.
@param lock_msg: message for afe on locking this host.
- @param max_wait: Max wait time in seconds.
+ @param max_wait: Max wait time in seconds to wait for duts to be idle.
@returns Boolean lock_success where True if all duts locked successfully or
False if we timed out waiting too long for hosts to go idle.
@@ -884,3 +896,91 @@
if not re.match('board:[^-]+-\d+', board):
return False
return True
+
+
+def _get_default_size_info(path):
+ """Get the default result size information.
+
+ In case directory summary is failed to build, assume the test result is not
+ throttled and all result sizes are the size of existing test results.
+
+ @return: A namedtuple of result size informations, including:
+ client_result_collected_KB: The total size (in KB) of test results
+ collected from test device. Set to be the total size of the
+ given path.
+ original_result_total_KB: The original size (in KB) of test results
+ before being trimmed. Set to be the total size of the given
+ path.
+ result_uploaded_KB: The total size (in KB) of test results to be
+ uploaded. Set to be the total size of the given path.
+ result_throttled: True if test results collection is throttled.
+ It's set to False in this default behavior.
+ """
+ total_size = file_utils.get_directory_size_kibibytes(path);
+ return result_utils_lib.ResultSizeInfo(
+ client_result_collected_KB=total_size,
+ original_result_total_KB=total_size,
+ result_uploaded_KB=total_size,
+ result_throttled=False)
+
+
+def _report_result_size_metrics(result_size_info):
+ """Report result sizes information to metrics.
+
+ @param result_size_info: A ResultSizeInfo namedtuple containing information
+ of test result sizes.
+ """
+ fields = {'result_throttled' : result_size_info.result_throttled}
+ metrics.Counter(RESULT_METRICS_PREFIX + 'client_result_collected_KB',
+ description='The total size (in KB) of test results '
+ 'collected from test device. Set to be the total size of '
+ 'the given path.'
+ ).increment_by(result_size_info.client_result_collected_KB,
+ fields=fields)
+ metrics.Counter(RESULT_METRICS_PREFIX + 'original_result_total_KB',
+ description='The original size (in KB) of test results '
+ 'before being trimmed.'
+ ).increment_by(result_size_info.original_result_total_KB,
+ fields=fields)
+ metrics.Counter(RESULT_METRICS_PREFIX + 'result_uploaded_KB',
+ description='The total size (in KB) of test results to be '
+ 'uploaded.'
+ ).increment_by(result_size_info.result_uploaded_KB,
+ fields=fields)
+
+
+def collect_result_sizes(path, log=logging.debug):
+ """Collect the result sizes information and build result summary.
+
+ It first tries to merge directory summaries and calculate the result sizes
+ including:
+ client_result_collected_KB: The volume in KB that's transfered from the test
+ device.
+ original_result_total_KB: The volume in KB that's the original size of the
+ result files before being trimmed.
+ result_uploaded_KB: The volume in KB that will be uploaded.
+ result_throttled: Indicating if the result files were throttled.
+
+ If directory summary merging failed for any reason, fall back to use the
+ total size of the given result directory.
+
+ @param path: Path of the result directory to get size information.
+ @param log: The logging method, default to logging.debug
+ @return: A ResultSizeInfo namedtuple containing information of test result
+ sizes.
+ """
+ try:
+ client_collected_bytes, summary = result_utils.merge_summaries(path)
+ result_size_info = result_utils_lib.get_result_size_info(
+ client_collected_bytes, summary)
+ html_file = os.path.join(path, result_view.DEFAULT_RESULT_SUMMARY_NAME)
+ result_view.build(client_collected_bytes, summary, html_file)
+ except:
+ log('Failed to calculate result sizes based on directory summaries for '
+ 'directory %s. Fall back to record the total size.\nException: %s' %
+ (path, traceback.format_exc()))
+ result_size_info = _get_default_size_info(path)
+
+ _report_result_size_metrics(result_size_info)
+
+ return result_size_info
\ No newline at end of file
diff --git a/tko/parse.py b/tko/parse.py
index 4a80aa3..4d8d4f1 100755
--- a/tko/parse.py
+++ b/tko/parse.py
@@ -13,10 +13,6 @@
import traceback
import common
-from autotest_lib.client.bin.result_tools import utils as result_utils
-from autotest_lib.client.bin.result_tools import utils_lib as result_utils_lib
-from autotest_lib.client.bin.result_tools import view as result_view
-from autotest_lib.client.common_lib import file_utils
from autotest_lib.client.common_lib import global_config
from autotest_lib.client.common_lib import mail, pidfile
from autotest_lib.client.common_lib import utils
@@ -35,25 +31,6 @@
'ParseOptions', ['reparse', 'mail_on_failure', 'dry_run', 'suite_report',
'datastore_creds', 'export_to_gcloud_path'])
-# Key names related to test result sizes to be stored in tko_job_keyvals.
-# The total size (in kB) of test results that generated during the test,
-# including:
-# * server side test logs and result files.
-# * client side test logs, sysinfo, system logs and crash dumps.
-# Note that a test can collect the same test result files from DUT multiple
-# times during the test, before and after each iteration/test. So the value of
-# CLIENT_RESULT_COLLECTED_KB could be larger than the value of
-# RESULT_UPLOADED_KB, which is the size of result directory on the server side,
-# even if the test result throttling is not applied.
-# The total size (in KB) of test results collected from test device.
-CLIENT_RESULT_COLLECTED_KB = 'client_result_collected_KB'
-# The original size (in KB) of test results before being trimmed.
-ORIGINAL_RESULT_TOTAL_KB = 'original_result_total_KB'
-# The total size (in KB) of test results to be uploaded by gs_offloader.
-RESULT_UPLOADED_KB = 'result_uploaded_KB'
-# Flag to indicate if test results collection is throttled.
-RESULT_THROTTLED = 'result_throttled'
-
def parse_args():
"""Parse args."""
# build up our options parser and parse sys.argv
@@ -255,53 +232,6 @@
tko_utils.dprint('DEBUG: Invalidated tests associated to job: ' + msg)
-def _get_result_sizes(path):
- """Get the result sizes information.
-
- It first tries to merge directory summaries and calculate the result sizes
- including:
- CLIENT_RESULT_COLLECTED_KB: The volume in KB that's transfered from the test
- device.
- ORIGINAL_RESULT_TOTAL_KB: The volume in KB that's the original size of the
- result files before being trimmed.
- RESULT_UPLOADED_KB: The volume in KB that will be uploaded.
- RESULT_THROTTLED: Indicating if the result files were throttled.
-
- If directory summary merging failed for any reason, fall back to use the
- total size of the given result directory.
-
- @param path: Path of the result directory to get size information.
- @return: A dictionary of result sizes information.
- """
- sizes = {}
- try:
- client_collected_bytes, summary = result_utils.merge_summaries(path)
- root_entry = summary[result_utils_lib.ROOT_DIR]
- sizes[CLIENT_RESULT_COLLECTED_KB] = client_collected_bytes / 1024
- sizes[ORIGINAL_RESULT_TOTAL_KB] = (
- root_entry[result_utils_lib.ORIGINAL_SIZE_BYTES]) / 1024
- sizes[RESULT_UPLOADED_KB] = (
- root_entry[result_utils_lib.TRIMMED_SIZE_BYTES])/ 1024
- # Test results are considered to be throttled if the total size of
- # results collected is different from the total size of trimmed results
- # from the client side.
- sizes[RESULT_THROTTLED] = (
- root_entry[result_utils_lib.ORIGINAL_SIZE_BYTES] !=
- root_entry[result_utils_lib.TRIMMED_SIZE_BYTES])
- html_file = os.path.join(path, result_view.DEFAULT_RESULT_SUMMARY_NAME)
- result_view.build(client_collected_bytes, summary, html_file)
- except:
- tko_utils.dprint('Failed to calculate result sizes based on directory '
- 'summaries. Fall back to record the total size.\n'
- 'Exception: %s' % traceback.format_exc())
- total_size = file_utils.get_directory_size_kibibytes(path);
- sizes[CLIENT_RESULT_COLLECTED_KB] = total_size
- sizes[ORIGINAL_RESULT_TOTAL_KB] = total_size
- sizes[RESULT_UPLOADED_KB] = total_size
- sizes[RESULT_THROTTLED] = 0
- return sizes
-
-
def parse_one(db, jobname, path, parse_options):
"""Parse a single job. Optionally send email on failure.
@@ -395,15 +325,18 @@
job.board = label_info.get('board', None)
job.suite = label_info.get('suite', None)
+ # Record test result size to job_keyvals
+ result_size_info = site_utils.collect_result_sizes(
+ path, log=tko_utils.dprint)
+ job.keyval_dict.update(result_size_info.__dict__)
+
# Upload job details to Sponge.
if not dry_run:
sponge_url = sponge_utils.upload_results(job, log=tko_utils.dprint)
if sponge_url:
job.keyval_dict['sponge_url'] = sponge_url
- # Record test result size to job_keyvals
- sizes = _get_result_sizes(path)
- job.keyval_dict.update(sizes)
+ # TODO(dshi): Update sizes with sponge_invocation.xml and throttle it.
# check for failures
message_lines = [""]
