| # SPDX-License-Identifier: Apache-2.0 |
| # |
| # Copyright (C) 2015, ARM Limited and contributors. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| |
| import argparse |
| import fnmatch as fnm |
| import json |
| import math |
| import numpy as np |
| import os |
| import re |
| import sys |
| import logging |
| |
| from collections import defaultdict |
| from colors import TestColors |
| |
| |
| |
| class Results(object): |
| |
| def __init__(self, results_dir): |
| self.results_dir = results_dir |
| self.results_json = results_dir + '/results.json' |
| self.results = {} |
| |
| # Setup logging |
| self._log = logging.getLogger('Results') |
| |
| # Do nothing if results have been already parsed |
| if os.path.isfile(self.results_json): |
| return |
| |
| # Parse results |
| self.base_wls = defaultdict(list) |
| self.test_wls = defaultdict(list) |
| |
| self._log.info('Loading energy/perf data...') |
| |
| for test_idx in sorted(os.listdir(self.results_dir)): |
| |
| test_dir = self.results_dir + '/' + test_idx |
| if not os.path.isdir(test_dir): |
| continue |
| |
| test = TestFactory.get(test_idx, test_dir, self.results) |
| test.parse() |
| |
| results_json = self.results_dir + '/results.json' |
| self._log.info('Dump perf results on JSON file [%s]...', |
| results_json) |
| with open(results_json, 'w') as outfile: |
| json.dump(self.results, outfile, indent=4, sort_keys=True) |
| |
| ################################################################################ |
| # Tests processing base classes |
| ################################################################################ |
| |
| class Test(object): |
| |
| def __init__(self, test_idx, test_dir, res): |
| self.test_idx = test_idx |
| self.test_dir = test_dir |
| self.res = res |
| match = TEST_DIR_RE.search(test_dir) |
| if not match: |
| self._log.error('Results folder not matching naming template') |
| self._log.error('Skip parsing of test results [%s]', test_dir) |
| return |
| |
| # Create required JSON entries |
| wtype = match.group(1) |
| if wtype not in res.keys(): |
| res[wtype] = {} |
| wload_idx = match.group(3) |
| if wload_idx not in res[wtype].keys(): |
| res[wtype][wload_idx] = {} |
| conf_idx = match.group(2) |
| if conf_idx not in res[wtype][wload_idx].keys(): |
| res[wtype][wload_idx][conf_idx] = {} |
| |
| # Set the workload type for this test |
| self.wtype = wtype |
| self.wload_idx = wload_idx |
| self.conf_idx = conf_idx |
| |
| # Energy metrics collected for all tests |
| self.little = [] |
| self.total = [] |
| self.big = [] |
| |
| def parse(self): |
| |
| self._log.info('Processing results from wtype [%s]', self.wtype) |
| |
| # Parse test's run results |
| for run_idx in sorted(os.listdir(self.test_dir)): |
| |
| # Skip all files which are not folders |
| run_dir = os.path.join(self.test_dir, run_idx) |
| if not os.path.isdir(run_dir): |
| continue |
| |
| run = self.parse_run(run_idx, run_dir) |
| self.collect_energy(run) |
| self.collect_performance(run) |
| |
| # Report energy/performance stats over all runs |
| self.res[self.wtype][self.wload_idx][self.conf_idx]\ |
| ['energy'] = self.energy() |
| self.res[self.wtype][self.wload_idx][self.conf_idx]\ |
| ['performance'] = self.performance() |
| |
| def collect_energy(self, run): |
| # Keep track of average energy of each run |
| self.little.append(run.little_nrg) |
| self.total.append(run.total_nrg) |
| self.big.append(run.big_nrg) |
| |
| def energy(self): |
| # Compute energy stats over all run |
| return { |
| 'LITTLE' : Stats(self.little).get(), |
| 'big' : Stats(self.big).get(), |
| 'Total' : Stats(self.total).get() |
| } |
| |
| class TestFactory(object): |
| |
| @staticmethod |
| def get(test_idx, test_dir, res): |
| |
| # Retrive workload class from results folder name |
| match = TEST_DIR_RE.search(test_dir) |
| if not match: |
| self._log.error('Results folder not matching naming template') |
| self._log.error('Skip parsing of test results [%s]', test_dir) |
| return |
| |
| # Create workload specifi test class |
| wtype = match.group(1) |
| |
| if wtype == 'rtapp': |
| return RTAppTest(test_idx, test_dir, res) |
| |
| # Return a generi test parser |
| return DefaultTest(test_idx, test_dir, res) |
| |
| class Energy(object): |
| |
| def __init__(self, nrg_file): |
| |
| # Set of exposed attributes |
| self.little = 0.0 |
| self.big = 0.0 |
| self.total = 0.0 |
| |
| self._log.debug('Parse [%s]...', nrg_file) |
| |
| with open(nrg_file, 'r') as infile: |
| nrg = json.load(infile) |
| |
| if 'LITTLE' in nrg: |
| self.little = float(nrg['LITTLE']) |
| if 'big' in nrg: |
| self.big = float(nrg['big']) |
| self.total = self.little + self.big |
| |
| self._log.debug('Energy LITTLE [%s], big [%s], Total [%s]', |
| self.little, self.big, self.total) |
| |
| class Stats(object): |
| |
| def __init__(self, data): |
| self.stats = {} |
| self.stats['count'] = len(data) |
| self.stats['min'] = min(data) |
| self.stats['max'] = max(data) |
| self.stats['avg'] = sum(data)/len(data) |
| std = Stats.stdev(data) |
| c99 = Stats.ci99(data, std) |
| self.stats['std'] = std |
| self.stats['c99'] = c99 |
| |
| def get(self): |
| return self.stats |
| |
| @staticmethod |
| def stdev(values): |
| sum1 = 0 |
| sum2 = 0 |
| for value in values: |
| sum1 += value |
| sum2 += math.pow(value, 2) |
| # print 'sum1: {}, sum2: {}'.format(sum1, sum2) |
| avg = sum1 / len(values) |
| var = (sum2 / len(values)) - (avg * avg) |
| # print 'avg: {} var: {}'.format(avg, var) |
| std = math.sqrt(var) |
| return float(std) |
| |
| @staticmethod |
| def ci99(values, std): |
| count = len(values) |
| ste = std / math.sqrt(count) |
| c99 = 2.58 * ste |
| return c99 |
| |
| |
| ################################################################################ |
| # Run processing base classes |
| ################################################################################ |
| |
| class Run(object): |
| |
| def __init__(self, run_idx, run_dir): |
| self.run_idx = run_idx |
| self.nrg = None |
| |
| self._log.debug('Parse [%s]...', 'Run', run_dir) |
| |
| # Energy stats |
| self.little_nrg = 0 |
| self.total_nrg = 0 |
| self.big_nrg = 0 |
| |
| nrg_file = run_dir + '/energy.json' |
| if os.path.isfile(nrg_file): |
| self.nrg = Energy(nrg_file) |
| self.little_nrg = self.nrg.little |
| self.total_nrg = self.nrg.total |
| self.big_nrg = self.nrg.big |
| |
| ################################################################################ |
| # RTApp workload parsing classes |
| ################################################################################ |
| |
| class RTAppTest(Test): |
| |
| def __init__(self, test_idx, test_dir, res): |
| super(RTAppTest, self).__init__(test_idx, test_dir, res) |
| |
| # RTApp specific performance metric |
| self.slack_pct = [] |
| self.perf_avg = [] |
| self.edp1 = [] |
| self.edp2 = [] |
| self.edp3 = [] |
| |
| self.rtapp_run = {} |
| |
| def parse_run(self, run_idx, run_dir): |
| return RTAppRun(run_idx, run_dir) |
| |
| def collect_performance(self, run): |
| # Keep track of average performances of each run |
| self.slack_pct.extend(run.slack_pct) |
| self.perf_avg.extend(run.perf_avg) |
| self.edp1.extend(run.edp1) |
| self.edp2.extend(run.edp2) |
| self.edp3.extend(run.edp3) |
| |
| # Keep track of performance stats for each run |
| self.rtapp_run[run.run_idx] = { |
| 'slack_pct' : Stats(run.slack_pct).get(), |
| 'perf_avg' : Stats(run.perf_avg).get(), |
| 'edp1' : Stats(run.edp1).get(), |
| 'edp2' : Stats(run.edp2).get(), |
| 'edp3' : Stats(run.edp3).get(), |
| } |
| |
| def performance(self): |
| |
| # Dump per run rtapp stats |
| prf_file = os.path.join(self.test_dir, 'performance.json') |
| with open(prf_file, 'w') as ofile: |
| json.dump(self.rtapp_run, ofile, indent=4, sort_keys=True) |
| |
| # Return oveall stats |
| return { |
| 'slack_pct' : Stats(self.slack_pct).get(), |
| 'perf_avg' : Stats(self.perf_avg).get(), |
| 'edp1' : Stats(self.edp1).get(), |
| 'edp2' : Stats(self.edp2).get(), |
| 'edp3' : Stats(self.edp3).get(), |
| } |
| |
| |
| class RTAppRun(Run): |
| |
| def __init__(self, run_idx, run_dir): |
| # Call base class to parse energy data |
| super(RTAppRun, self).__init__(run_idx, run_dir) |
| |
| # RTApp specific performance stats |
| self.slack_pct = [] |
| self.perf_avg = [] |
| self.edp1 = [] |
| self.edp2 = [] |
| self.edp3 = [] |
| |
| rta = {} |
| |
| # Load run's performance of each task |
| for task_idx in sorted(os.listdir(run_dir)): |
| |
| if not fnm.fnmatch(task_idx, 'rt-app-*.log'): |
| continue |
| |
| # Parse run's performance results |
| prf_file = run_dir + '/' + task_idx |
| task = RTAppPerf(prf_file, self.nrg) |
| |
| # Keep track of average performances of each task |
| self.slack_pct.append(task.prf['slack_pct']) |
| self.perf_avg.append(task.prf['perf_avg']) |
| self.edp1.append(task.prf['edp1']) |
| self.edp2.append(task.prf['edp2']) |
| self.edp3.append(task.prf['edp3']) |
| |
| # Keep track of performance stats for each task |
| rta[task.name] = task.prf |
| |
| # Dump per task rtapp stats |
| prf_file = os.path.join(run_dir, 'performance.json') |
| with open(prf_file, 'w') as ofile: |
| json.dump(rta, ofile, indent=4, sort_keys=True) |
| |
| |
| class RTAppPerf(object): |
| |
| def __init__(self, perf_file, nrg): |
| |
| # Set of exposed attibutes |
| self.prf = { |
| 'perf_avg' : 0, |
| 'perf_std' : 0, |
| 'run_sum' : 0, |
| 'slack_sum' : 0, |
| 'slack_pct' : 0, |
| 'edp1' : 0, |
| 'edp2' : 0, |
| 'edp3' : 0 |
| } |
| |
| self._log.debug('Parse [%s]...', perf_file) |
| |
| # Load performance data for each RT-App task |
| self.name = perf_file.split('-')[-2] |
| self.data = np.loadtxt(perf_file, comments='#', unpack=False) |
| |
| # Max Slack (i.e. configured/expected slack): period - run |
| max_slack = np.subtract( |
| self.data[:,RTAPP_COL_C_PERIOD], self.data[:,RTAPP_COL_C_RUN]) |
| |
| # Performance Index: 100 * slack / max_slack |
| perf = np.divide(self.data[:,RTAPP_COL_SLACK], max_slack) |
| perf = np.multiply(perf, 100) |
| self.prf['perf_avg'] = np.mean(perf) |
| self.prf['perf_std'] = np.std(perf) |
| self._log.debug('perf [%s]: %6.2f,%6.2f', |
| self.name, self.prf['perf_avg'], |
| self.prf['perf_std']) |
| |
| # Negative slacks |
| nslacks = self.data[:,RTAPP_COL_SLACK] |
| nslacks = nslacks[nslacks < 0] |
| self._log.debug('Negative slacks: %s', nslacks) |
| self.prf['slack_sum'] = -nslacks.sum() |
| self._log.debug('Negative slack [%s] sum: %6.2f', |
| self.name, self.prf['slack_sum']) |
| |
| # Slack over run-time |
| self.prf['run_sum'] = np.sum(self.data[:,RTAPP_COL_RUN]) |
| self.prf['slack_pct'] = 100 * self.prf['slack_sum'] / self.prf['run_sum'] |
| self._log.debug('SlackPct [%s]: %6.2f %%', self.name, self.slack_pct) |
| |
| if nrg is None: |
| return |
| |
| # Computing EDP |
| self.prf['edp1'] = nrg.total * math.pow(self.prf['run_sum'], 1) |
| self._log.debug('EDP1 [%s]: {%6.2f}', self.name, self.prf['edp1']) |
| self.prf['edp2'] = nrg.total * math.pow(self.prf['run_sum'], 2) |
| self._log.debug('EDP2 [%s]: %6.2f', self.name, self.prf['edp2']) |
| self.prf['edp3'] = nrg.total * math.pow(self.prf['run_sum'], 3) |
| self._log.debug('EDP3 [%s]: %6.2f', self.name, self.prf['edp3']) |
| |
| |
| # Columns of the per-task rt-app log file |
| RTAPP_COL_IDX = 0 |
| RTAPP_COL_PERF = 1 |
| RTAPP_COL_RUN = 2 |
| RTAPP_COL_PERIOD = 3 |
| RTAPP_COL_START = 4 |
| RTAPP_COL_END = 5 |
| RTAPP_COL_REL_ST = 6 |
| RTAPP_COL_SLACK = 7 |
| RTAPP_COL_C_RUN = 8 |
| RTAPP_COL_C_PERIOD = 9 |
| RTAPP_COL_WU_LAT = 10 |
| |
| ################################################################################ |
| # Generic workload performance parsing class |
| ################################################################################ |
| |
| class DefaultTest(Test): |
| |
| def __init__(self, test_idx, test_dir, res): |
| super(DefaultTest, self).__init__(test_idx, test_dir, res) |
| |
| # Default performance metric |
| self.ctime_avg = [] |
| self.perf_avg = [] |
| self.edp1 = [] |
| self.edp2 = [] |
| self.edp3 = [] |
| |
| def parse_run(self, run_idx, run_dir): |
| return DefaultRun(run_idx, run_dir) |
| |
| def collect_performance(self, run): |
| # Keep track of average performances of each run |
| self.ctime_avg.append(run.ctime_avg) |
| self.perf_avg.append(run.perf_avg) |
| self.edp1.append(run.edp1) |
| self.edp2.append(run.edp2) |
| self.edp3.append(run.edp3) |
| |
| def performance(self): |
| return { |
| 'ctime_avg' : Stats(self.ctime_avg).get(), |
| 'perf_avg' : Stats(self.perf_avg).get(), |
| 'edp1' : Stats(self.edp1).get(), |
| 'edp2' : Stats(self.edp2).get(), |
| 'edp3' : Stats(self.edp3).get(), |
| } |
| |
| class DefaultRun(Run): |
| |
| def __init__(self, run_idx, run_dir): |
| # Call base class to parse energy data |
| super(DefaultRun, self).__init__(run_idx, run_dir) |
| |
| # Default specific performance stats |
| self.ctime_avg = 0 |
| self.perf_avg = 0 |
| self.edp1 = 0 |
| self.edp2 = 0 |
| self.edp3 = 0 |
| |
| # Load default performance.json |
| prf_file = os.path.join(run_dir, 'performance.json') |
| if not os.path.isfile(prf_file): |
| self._log.warning('No performance.json found in %s', |
| run_dir) |
| return |
| |
| # Load performance report from JSON |
| with open(prf_file, 'r') as infile: |
| prf = json.load(infile) |
| |
| # Keep track of performance value |
| self.ctime_avg = prf['ctime'] |
| self.perf_avg = prf['performance'] |
| |
| # Compute EDP indexes if energy measurements are available |
| if self.nrg is None: |
| return |
| |
| # Computing EDP |
| self.edp1 = self.nrg.total * math.pow(self.ctime_avg, 1) |
| self.edp2 = self.nrg.total * math.pow(self.ctime_avg, 2) |
| self.edp3 = self.nrg.total * math.pow(self.ctime_avg, 3) |
| |
| |
| ################################################################################ |
| # Globals |
| ################################################################################ |
| |
| # Regexp to match the format of a result folder |
| TEST_DIR_RE = re.compile( |
| r'.*/([^:]*):([^:]*):([^:]*)' |
| ) |
| |
| #vim :set tabstop=4 shiftwidth=4 expandtab |
