| # SPDX-License-Identifier: Apache-2.0 |
| # |
| # Copyright (C) 2015, ARM Limited and contributors. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| |
| import argparse |
| import fnmatch as fnm |
| import json |
| import math |
| import numpy as np |
| import os |
| import re |
| import sys |
| import logging |
| |
| from collections import defaultdict |
| from colors import TestColors |
| from results import Results |
| |
| |
| # By default compare all the possible combinations |
| DEFAULT_COMPARE = [(r'base_', r'test_')] |
| |
| class Report(object): |
| |
| |
| def __init__(self, results_dir, compare=None, formats=['relative']): |
| self.results_json = results_dir + '/results.json' |
| self.results = {} |
| |
| self.compare = [] |
| |
| # Setup logging |
| self._log = logging.getLogger('Report') |
| |
| # Parse results (if required) |
| if not os.path.isfile(self.results_json): |
| Results(results_dir) |
| |
| # Load results from file (if already parsed) |
| self._log.info('Load results from [%s]...', |
| self.results_json) |
| with open(self.results_json) as infile: |
| self.results = json.load(infile) |
| |
| # Setup configuration comparisons |
| if compare is None: |
| compare = DEFAULT_COMPARE |
| self._log.warning('Comparing all the possible combination') |
| for (base_rexp, test_rexp) in compare: |
| self._log.info('Configured regexps for comparisions ' |
| '(bases , tests): (%s, %s)', |
| base_rexp, test_rexp) |
| base_rexp = re.compile(base_rexp, re.DOTALL) |
| test_rexp = re.compile(test_rexp, re.DOTALL) |
| self.compare.append((base_rexp, test_rexp)) |
| |
| # Report all supported workload classes |
| self.__rtapp_report(formats) |
| self.__default_report(formats) |
| |
| ############################### REPORT RTAPP ############################### |
| |
| def __rtapp_report(self, formats): |
| |
| if 'rtapp' not in self.results.keys(): |
| self._log.debug('No RTApp workloads to report') |
| return |
| |
| self._log.debug('Reporting RTApp workloads') |
| |
| # Setup lables depending on requested report |
| if 'absolute' in formats: |
| nrg_lable = 'Energy Indexes (Absolute)' |
| prf_lable = 'Performance Indexes (Absolute)' |
| self._log.info('') |
| self._log.info('Absolute comparisions:') |
| print '' |
| else: |
| nrg_lable = 'Energy Indexes (Relative)' |
| prf_lable = 'Performance Indexes (Relative)' |
| self._log.info('') |
| self._log.info('Relative comparisions:') |
| print '' |
| |
| # Dump headers |
| print '{:13s} {:20s} |'\ |
| ' {:33s} | {:54s} |'\ |
| .format('Test Id', 'Comparision', |
| nrg_lable, prf_lable) |
| print '{:13s} {:20s} |'\ |
| ' {:>10s} {:>10s} {:>10s} |'\ |
| ' {:>10s} {:>10s} {:>10s} {:>10s} {:>10s} |'\ |
| .format('', '', |
| 'LITTLE', 'big', 'Total', |
| 'PerfIndex', 'NegSlacks', 'EDP1', 'EDP2', 'EDP3') |
| |
| # For each test |
| _results = self.results['rtapp'] |
| for tid in sorted(_results.keys()): |
| new_test = True |
| # For each configuration... |
| for base_idx in sorted(_results[tid].keys()): |
| # Which matches at least on base regexp |
| for (base_rexp, test_rexp) in self.compare: |
| if not base_rexp.match(base_idx): |
| continue |
| # Look for a configuration which matches the test regexp |
| for test_idx in sorted(_results[tid].keys()): |
| if test_idx == base_idx: |
| continue |
| if new_test: |
| print '{:-<37s}+{:-<35s}+{:-<56s}+'\ |
| .format('','', '') |
| self.__rtapp_reference(tid, base_idx) |
| new_test = False |
| if test_rexp.match(test_idx) == None: |
| continue |
| self.__rtapp_compare(tid, base_idx, test_idx, formats) |
| |
| print '' |
| |
| def __rtapp_reference(self, tid, base_idx): |
| _results = self.results['rtapp'] |
| |
| self._log.debug('Test %s: compare against [%s] base', |
| tid, base_idx) |
| res_line = '{0:12s}: {1:22s} | '.format(tid, base_idx) |
| |
| # Dump all energy metrics |
| for cpus in ['LITTLE', 'big', 'Total']: |
| res_base = _results[tid][base_idx]['energy'][cpus]['avg'] |
| # Dump absolute values |
| res_line += ' {0:10.3f}'.format(res_base) |
| res_line += ' |' |
| |
| # If available, dump also performance results |
| if 'performance' not in _results[tid][base_idx].keys(): |
| print res_line |
| return |
| |
| for pidx in ['perf_avg', 'slack_pct', 'edp1', 'edp2', 'edp3']: |
| res_base = _results[tid][base_idx]['performance'][pidx]['avg'] |
| |
| self._log.debug('idx: %s, base: %s', pidx, res_base) |
| |
| if pidx in ['perf_avg']: |
| res_line += ' {0:s}'.format(TestColors.rate(res_base)) |
| continue |
| if pidx in ['slack_pct']: |
| res_line += ' {0:s}'.format( |
| TestColors.rate(res_base, positive_is_good = False)) |
| continue |
| if 'edp' in pidx: |
| res_line += ' {0:10.2e}'.format(res_base) |
| continue |
| res_line += ' |' |
| print res_line |
| |
| def __rtapp_compare(self, tid, base_idx, test_idx, formats): |
| _results = self.results['rtapp'] |
| |
| self._log.debug('Test %s: compare %s with %s', |
| tid, base_idx, test_idx) |
| res_line = '{0:12s}: {1:20s} | '.format(tid, test_idx) |
| |
| # Dump all energy metrics |
| for cpus in ['LITTLE', 'big', 'Total']: |
| res_base = _results[tid][base_idx]['energy'][cpus]['avg'] |
| res_test = _results[tid][test_idx]['energy'][cpus]['avg'] |
| speedup_cnt = res_test - res_base |
| if 'absolute' in formats: |
| res_line += ' {0:10.2f}'.format(speedup_cnt) |
| else: |
| speedup_pct = 0 |
| if res_base != 0: |
| speedup_pct = 100.0 * speedup_cnt / res_base |
| res_line += ' {0:s}'\ |
| .format(TestColors.rate( |
| speedup_pct, |
| positive_is_good = False)) |
| res_line += ' |' |
| |
| # If available, dump also performance results |
| if 'performance' not in _results[tid][base_idx].keys(): |
| print res_line |
| return |
| |
| for pidx in ['perf_avg', 'slack_pct', 'edp1', 'edp2', 'edp3']: |
| res_base = _results[tid][base_idx]['performance'][pidx]['avg'] |
| res_test = _results[tid][test_idx]['performance'][pidx]['avg'] |
| |
| self._log.debug('idx: %s, base: %s, test: %s', |
| pidx, res_base, res_test) |
| |
| if pidx in ['perf_avg']: |
| res_line += ' {0:s}'.format(TestColors.rate(res_test)) |
| continue |
| |
| if pidx in ['slack_pct']: |
| res_line += ' {0:s}'.format( |
| TestColors.rate(res_test, positive_is_good = False)) |
| continue |
| |
| # Compute difference base-vs-test |
| if 'edp' in pidx: |
| speedup_cnt = res_base - res_test |
| if 'absolute': |
| res_line += ' {0:10.2e}'.format(speedup_cnt) |
| else: |
| res_line += ' {0:s}'.format(TestColors.rate(speedup_pct)) |
| |
| res_line += ' |' |
| print res_line |
| |
| ############################### REPORT DEFAULT ############################# |
| |
| def __default_report(self, formats): |
| |
| # Build list of workload types which can be rendered using the default parser |
| wtypes = [] |
| for supported_wtype in DEFAULT_WTYPES: |
| if supported_wtype in self.results.keys(): |
| wtypes.append(supported_wtype) |
| |
| if len(wtypes) == 0: |
| self._log.debug('No Default workloads to report') |
| return |
| |
| self._log.debug('Reporting Default workloads') |
| |
| # Setup lables depending on requested report |
| if 'absolute' in formats: |
| nrg_lable = 'Energy Indexes (Absolute)' |
| prf_lable = 'Performance Indexes (Absolute)' |
| self._log.info('') |
| self._log.info('Absolute comparisions:') |
| print '' |
| else: |
| nrg_lable = 'Energy Indexes (Relative)' |
| prf_lable = 'Performance Indexes (Relative)' |
| self._log.info('') |
| self._log.info('Relative comparisions:') |
| print '' |
| |
| # Dump headers |
| print '{:9s} {:20s} |'\ |
| ' {:33s} | {:54s} |'\ |
| .format('Test Id', 'Comparision', |
| nrg_lable, prf_lable) |
| print '{:9s} {:20s} |'\ |
| ' {:>10s} {:>10s} {:>10s} |'\ |
| ' {:>10s} {:>10s} {:>10s} {:>10s} {:>10s} |'\ |
| .format('', '', |
| 'LITTLE', 'big', 'Total', |
| 'Perf', 'CTime', 'EDP1', 'EDP2', 'EDP3') |
| |
| # For each default test |
| for wtype in wtypes: |
| _results = self.results[wtype] |
| for tid in sorted(_results.keys()): |
| new_test = True |
| # For each configuration... |
| for base_idx in sorted(_results[tid].keys()): |
| # Which matches at least on base regexp |
| for (base_rexp, test_rexp) in self.compare: |
| if not base_rexp.match(base_idx): |
| continue |
| # Look for a configuration which matches the test regexp |
| for test_idx in sorted(_results[tid].keys()): |
| if test_idx == base_idx: |
| continue |
| if new_test: |
| print '{:-<37s}+{:-<35s}+{:-<56s}+'\ |
| .format('','', '') |
| new_test = False |
| if not test_rexp.match(test_idx): |
| continue |
| self.__default_compare(wtype, tid, base_idx, test_idx, formats) |
| |
| print '' |
| |
| def __default_compare(self, wtype, tid, base_idx, test_idx, formats): |
| _results = self.results[wtype] |
| |
| self._log.debug('Test %s: compare %s with %s', |
| tid, base_idx, test_idx) |
| res_comp = '{0:s} vs {1:s}'.format(test_idx, base_idx) |
| res_line = '{0:8s}: {1:22s} | '.format(tid, res_comp) |
| |
| # Dump all energy metrics |
| for cpus in ['LITTLE', 'big', 'Total']: |
| |
| # If either base of test have a 0 MAX energy, this measn that |
| # energy has not been collected |
| base_max = _results[tid][base_idx]['energy'][cpus]['max'] |
| test_max = _results[tid][test_idx]['energy'][cpus]['max'] |
| if base_max == 0 or test_max == 0: |
| res_line += ' {0:10s}'.format('NA') |
| continue |
| |
| # Otherwise, report energy values |
| res_base = _results[tid][base_idx]['energy'][cpus]['avg'] |
| res_test = _results[tid][test_idx]['energy'][cpus]['avg'] |
| |
| speedup_cnt = res_test - res_base |
| if 'absolute' in formats: |
| res_line += ' {0:10.2f}'.format(speedup_cnt) |
| else: |
| speedup_pct = 100.0 * speedup_cnt / res_base |
| res_line += ' {0:s}'\ |
| .format(TestColors.rate( |
| speedup_pct, |
| positive_is_good = False)) |
| res_line += ' |' |
| |
| # If available, dump also performance results |
| if 'performance' not in _results[tid][base_idx].keys(): |
| print res_line |
| return |
| |
| for pidx in ['perf_avg', 'ctime_avg', 'edp1', 'edp2', 'edp3']: |
| res_base = _results[tid][base_idx]['performance'][pidx]['avg'] |
| res_test = _results[tid][test_idx]['performance'][pidx]['avg'] |
| |
| self._log.debug('idx: %s, base: %s, test: %s', |
| pidx, res_base, res_test) |
| |
| # Compute difference base-vs-test |
| speedup_cnt = 0 |
| if res_base != 0: |
| if pidx in ['perf_avg']: |
| speedup_cnt = res_test - res_base |
| else: |
| speedup_cnt = res_base - res_test |
| |
| # Compute speedup if required |
| speedup_pct = 0 |
| if 'absolute' in formats: |
| if 'edp' in pidx: |
| res_line += ' {0:10.2e}'.format(speedup_cnt) |
| else: |
| res_line += ' {0:10.2f}'.format(speedup_cnt) |
| else: |
| if res_base != 0: |
| if pidx in ['perf_avg']: |
| # speedup_pct = 100.0 * speedup_cnt / res_base |
| speedup_pct = speedup_cnt |
| else: |
| speedup_pct = 100.0 * speedup_cnt / res_base |
| res_line += ' {0:s}'.format(TestColors.rate(speedup_pct)) |
| res_line += ' |' |
| print res_line |
| |
| # List of workload types which can be parsed using the default test parser |
| DEFAULT_WTYPES = ['perf_bench_messaging', 'perf_bench_pipe'] |
| |
| #vim :set tabstop=4 shiftwidth=4 expandtab |
