user_activity: Added implementation for the metrics experiment.
The validation of the benchmark metrics is done on a pair of CWP
function sets.
BUG=None
TEST=benchmark_metrics_validation_unittest passes
Change-Id: I15cb5976ba4d9a41ff97376b0ac3320e9a798fd5
Reviewed-on: https://chrome-internal-review.googlesource.com/286160
Reviewed-by: Luis Lozano <llozano@chromium.org>
Tested-by: Evelina Dumitrescu <evelinad@google.com>
Reviewed-on: https://chromium-review.googlesource.com/435911
Commit-Ready: Luis Lozano <llozano@chromium.org>
Tested-by: Luis Lozano <llozano@chromium.org>
diff --git a/user_activity_benchmarks/benchmark_metrics_experiment.py b/user_activity_benchmarks/benchmark_metrics_experiment.py
new file mode 100755
index 0000000..9c04a89
--- /dev/null
+++ b/user_activity_benchmarks/benchmark_metrics_experiment.py
@@ -0,0 +1,307 @@
+#!/usr/bin/python2
+#
+# Copyright 2016 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Runs an experiment with the benchmark metrics on a pair of CWP data sets.
+
+A data set should contain the files with the pairwise inclusive and the
+inclusive statistics. The pairwise inclusive file contains pairs of
+parent and child functions with their inclusive count fractions out of the
+total amount of inclusive count values and the files of the child functions.
+The inclusive file contains the functions with their inclusive count fraction
+out of the total amount of inclusive count values and the file name of the
+function. The input data should be collected using the scripts
+collect_experiment_data.sh or collect_experiment_data_odd_even_session.sh
+
+For every function, this script computes the distance and the score values.
+The output is stored in the file cwp_functions_statistics_file.
+
+For every Chrome OS component, this script computes a set of metrics consisting
+in the number of functions, the average and cumulative distance and score of
+the functions matching the group. The output is stored in the file
+cwp_function_groups_statistics_file.
+"""
+
+import argparse
+from collections import defaultdict
+import csv
+import os
+import sys
+import benchmark_metrics
+
+
+class MetricsExperiment(object):
+ """Runs an experiment with the benchmark metrics on a pair of data sets."""
+
+ def __init__(self, cwp_pairwise_inclusive_reference,
+ cwp_pairwise_inclusive_test, cwp_inclusive_reference,
+ cwp_inclusive_test, cwp_function_groups_file,
+ cwp_function_groups_statistics_file,
+ cwp_function_statistics_file):
+ """Initializes the MetricsExperiment class.
+
+ Args:
+ cwp_pairwise_inclusive_reference: The CSV file containing the pairwise
+ inclusive values from the reference data set.
+ cwp_pairwise_inclusive_test: The CSV file containing the pairwise
+ inclusive values from the test data set.
+ cwp_inclusive_reference: The CSV file containing the inclusive values
+ from the reference data set.
+ cwp_inclusive_test: The CSV file containing the inclusive values from
+ the test data set.
+ cwp_function_groups_file: The CSV file containing the groups of functions.
+ cwp_function_groups_statistics_file: The output CSV file that will
+ contain the metrics for the function groups.
+ cwp_function_statistics_file: The output CSV file that will contain the
+ metrics for the CWP functions.
+ """
+ self._cwp_pairwise_inclusive_reference = cwp_pairwise_inclusive_reference
+ self._cwp_pairwise_inclusive_test = cwp_pairwise_inclusive_test
+ self._cwp_inclusive_reference = cwp_inclusive_reference
+ self._cwp_inclusive_test = cwp_inclusive_test
+ self._cwp_function_groups_file = cwp_function_groups_file
+ self._cwp_function_groups_statistics_file = \
+ cwp_function_groups_statistics_file
+ self._cwp_function_statistics_file = cwp_function_statistics_file
+
+ @staticmethod
+ def ParsePairwiseInclusiveStatisticsFile(file_name):
+ """Parses the pairwise inclusive statistics files.
+
+ A line of the file should contain a pair of a parent and a child function,
+ concatenated by a ;;, the name of the file where the child function is
+ defined and the inclusive count fractions of the pair of functions out of
+ the total amount of inclusive count values.
+
+ Args:
+ file_name: The file containing the pairwise inclusive statistics of the
+ CWP functions.
+
+ Returns:
+ A dict containing the statistics of the parent functions and each of
+ their child functions. The key of the dict is the name of the parent
+ function. The value is a dict having as a key the name of the child
+ function with its file name separated by a ',' and as a value the
+ inclusive count fraction of the child function.
+ """
+ pairwise_inclusive_statistics = defaultdict(lambda: defaultdict(float))
+
+ with open(file_name) as \
+ pairwise_inclusive_statistics_file:
+ statistics_reader = csv.DictReader(
+ pairwise_inclusive_statistics_file, delimiter=',')
+ for statistic in statistics_reader:
+ parent_function_name, child_function_name = \
+ statistic['parent_child_functions'].split(';;')
+ child_function_file_name = \
+ os.path.normpath(statistic['child_function_file'])
+ inclusive_count_fraction = \
+ float(statistic['inclusive_count_fraction'])
+
+ if all([parent_function_name, child_function_name, \
+ inclusive_count_fraction]):
+
+ # There might be situations where a child function appears in
+ # multiple files or objects. Such situations can occur when in the
+ # Dremel queries there are not specified the Chrome OS version and the
+ # name of the board (i.e the files can belong to different kernel or
+ # library versions), when the child function is a template function
+ # that is declared in a header file or there are name collisions
+ # between multiple executable objects.
+ # If a pair of child and parent functions appears multiple times, we
+ # add their inclusive count values.
+ child_function_key = ','.join([child_function_name,
+ child_function_file_name])
+ pairwise_inclusive_statistics[parent_function_name]\
+ [child_function_key] += inclusive_count_fraction
+
+ return pairwise_inclusive_statistics
+
+ @staticmethod
+ def ParseInclusiveStatisticsFile(inclusive_statistics_file_name):
+ """Parses the inclusive statistics files.
+
+ Args:
+ inclusive_statistics_file_name: The file containing the inclusive
+ statistics of the CWP functions.
+
+ Returns:
+ A dict having as a key the function name and file where the function is
+ defined separated by a ',' and as a value the inclusive count fraction.
+ """
+ inclusive_statistics = defaultdict(float)
+
+ with open(inclusive_statistics_file_name) as inclusive_statistics_file:
+ statistics_reader = \
+ csv.DictReader(inclusive_statistics_file, delimiter=',')
+
+ for statistic in statistics_reader:
+ function_name = statistic['function']
+ file_name = os.path.normpath(statistic['file'])
+ inclusive_count_fraction = \
+ float(statistic['inclusive_count_fraction'])
+
+ # There might be situations where a function appears in multiple files
+ # or objects. Such situations can occur when in the Dremel queries there
+ # are not specified the Chrome OS version and the name of the board (i.e
+ # the files can belong to different kernel or library versions).
+ if all([function_name, file_name, inclusive_count_fraction]):
+ parent_function_key = ','.join([function_name, file_name])
+ inclusive_statistics[parent_function_key] += inclusive_count_fraction
+
+ return inclusive_statistics
+
+ def PerformComputation(self):
+ """Does the benchmark metrics experimental computation.
+
+ For every function, it is computed a distance based on the sum of the
+ differences of the fractions spent in the child functions. Afterwards,
+ it is computed a score based on the inclusive values fractions and the
+ distance value. The statistics for all the function are written in the file
+ self._cwp_function_statistics_file.
+
+ The functions are grouped on Chrome OS components based on the path of the
+ file where a function is defined. For every group, there are computed the
+ total number of functions matching that group, the cumulative distance, the
+ average distance and the cumulative score of the functions.
+ """
+
+ inclusive_statistics_reference = \
+ self.ParseInclusiveStatisticsFile(self._cwp_inclusive_reference)
+ inclusive_statistics_test = \
+ self.ParseInclusiveStatisticsFile(self._cwp_inclusive_test)
+ pairwise_inclusive_statistics_reference = \
+ self.ParsePairwiseInclusiveStatisticsFile(
+ self._cwp_pairwise_inclusive_reference)
+ pairwise_inclusive_statistics_test = \
+ self.ParsePairwiseInclusiveStatisticsFile(
+ self._cwp_pairwise_inclusive_test)
+ parent_function_statistics = {}
+
+ with open(self._cwp_function_groups_file, 'r') as input_file:
+ cwp_function_groups = [line.split() for line in input_file]
+
+ for parent_function_key, parent_function_fraction_test \
+ in inclusive_statistics_test.iteritems():
+ parent_function_name, parent_function_file_name = \
+ parent_function_key.split(',')
+
+ parent_function_fraction_reference = \
+ inclusive_statistics_reference.get(parent_function_key, 0.0)
+
+ child_functions_statistics_test = \
+ pairwise_inclusive_statistics_test.get(parent_function_name, {})
+
+ child_functions_statistics_reference = \
+ pairwise_inclusive_statistics_reference.get(parent_function_name, {})
+
+ distance = benchmark_metrics.ComputeDistanceForFunction(
+ child_functions_statistics_test, child_functions_statistics_reference)
+
+ parent_function_score_test = benchmark_metrics.ComputeScoreForFunction(
+ distance, parent_function_fraction_test,
+ parent_function_fraction_reference)
+
+ parent_function_statistics[parent_function_key] = \
+ (distance, parent_function_score_test)
+
+ with open(self._cwp_function_statistics_file, 'w') as output_file:
+ statistics_lines = ['function,file,distance,score']
+ statistics_lines += \
+ [','.join([parent_function_key.replace(';;', ','),
+ str(statistic[0]),
+ str(statistic[1])])
+ for parent_function_key, statistic
+ in parent_function_statistics.iteritems()]
+ output_file.write('\n'.join(statistics_lines))
+
+ cwp_groups_statistics_test = benchmark_metrics.ComputeMetricsForComponents(
+ cwp_function_groups, parent_function_statistics)
+
+ with open(self._cwp_function_groups_statistics_file, 'w') as output_file:
+ group_statistics_lines = \
+ ['group,file_path,function_count,distance_cum,distance_avg,score_cum,'
+ 'score_avg']
+ group_statistics_lines += \
+ [','.join([group_name,
+ str(statistic[0]),
+ str(statistic[1]),
+ str(statistic[2]),
+ str(statistic[3]),
+ str(statistic[4]),
+ str(statistic[5])])
+ for group_name, statistic
+ in cwp_groups_statistics_test.iteritems()]
+ output_file.write('\n'.join(group_statistics_lines))
+
+
+def ParseArguments(arguments):
+ parser = argparse.ArgumentParser(
+ description='Runs an experiment with the benchmark metrics on a pair of '
+ 'CWP data sets.')
+ parser.add_argument(
+ '--cwp_pairwise_inclusive_reference',
+ required=True,
+ help='The reference CSV file that will contain a pair of parent and '
+ 'child functions with their inclusive count fractions out of the total '
+ 'amount of inclusive count values.')
+ parser.add_argument(
+ '--cwp_pairwise_inclusive_test',
+ required=True,
+ help='The test CSV file that will contain a pair of parent and '
+ 'child functions with their inclusive count fractions out of the total '
+ 'amount of inclusive count values.')
+ parser.add_argument(
+ '--cwp_inclusive_reference',
+ required=True,
+ help='The reference CSV file that will contain a function with its '
+ 'inclusive count fraction out of the total amount of inclusive count '
+ 'values.')
+ parser.add_argument(
+ '--cwp_inclusive_test',
+ required=True,
+ help='The test CSV file that will contain a function with its '
+ 'inclusive count fraction out of the total amount of inclusive count '
+ 'values.')
+ parser.add_argument(
+ '-g',
+ '--cwp_function_groups_file',
+ required=True,
+ help='The file that will contain the CWP function groups.'
+ 'A line consists in the group name and a file path. A group must '
+ 'represent a ChromeOS component.')
+ parser.add_argument(
+ '-s',
+ '--cwp_function_groups_statistics_file',
+ required=True,
+ help='The output file that will contain the metric statistics for the '
+ 'CWP function groups in CSV format. A line consists in the group name, '
+ 'file path, number of functions matching the group, the total score '
+ 'and distance values.')
+ parser.add_argument(
+ '-f',
+ '--cwp_function_statistics_file',
+ required=True,
+ help='The output file that will contain the metric statistics for the '
+ 'CWP functions in CSV format. A line consists in the function name, file '
+ 'name, cummulative distance, average distance, cummulative score and '
+ 'average score values.')
+
+ options = parser.parse_args(arguments)
+ return options
+
+
+def Main(argv):
+ options = ParseArguments(argv)
+ metrics_experiment = MetricsExperiment(
+ options.cwp_pairwise_inclusive_reference,
+ options.cwp_pairwise_inclusive_test, options.cwp_inclusive_reference,
+ options.cwp_inclusive_test, options.cwp_function_groups_file,
+ options.cwp_function_groups_statistics_file,
+ options.cwp_function_statistics_file)
+ metrics_experiment.PerformComputation()
+
+
+if __name__ == '__main__':
+ Main(sys.argv[1:])
