|
#!/usr/local/bin/python -O |
|
|
|
""" A Python Benchmark Suite |
|
|
|
""" |
|
# |
|
# Note: Please keep this module compatible to Python 1.5.2. |
|
# |
|
# Tests may include features in later Python versions, but these |
|
# should then be embedded in try-except clauses in the configuration |
|
# module Setup.py. |
|
# |
|
|
|
# pybench Copyright |
|
__copyright__ = """\ |
|
Copyright (c), 1997-2006, Marc-Andre Lemburg (mal@lemburg.com) |
|
Copyright (c), 2000-2006, eGenix.com Software GmbH (info@egenix.com) |
|
|
|
All Rights Reserved. |
|
|
|
Permission to use, copy, modify, and distribute this software and its |
|
documentation for any purpose and without fee or royalty is hereby |
|
granted, provided that the above copyright notice appear in all copies |
|
and that both that copyright notice and this permission notice appear |
|
in supporting documentation or portions thereof, including |
|
modifications, that you make. |
|
|
|
THE AUTHOR MARC-ANDRE LEMBURG DISCLAIMS ALL WARRANTIES WITH REGARD TO |
|
THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND |
|
FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, |
|
INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING |
|
FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, |
|
NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION |
|
WITH THE USE OR PERFORMANCE OF THIS SOFTWARE ! |
|
""" |
|
|
|
import sys, time, operator, string, platform |
|
from CommandLine import * |
|
|
|
try: |
|
import cPickle |
|
pickle = cPickle |
|
except ImportError: |
|
import pickle |
|
|
|
# Version number; version history: see README file ! |
|
__version__ = '2.0' |
|
|
|
### Constants |
|
|
|
# Second fractions |
|
MILLI_SECONDS = 1e3 |
|
MICRO_SECONDS = 1e6 |
|
|
|
# Percent unit |
|
PERCENT = 100 |
|
|
|
# Horizontal line length |
|
LINE = 79 |
|
|
|
# Minimum test run-time |
|
MIN_TEST_RUNTIME = 1e-3 |
|
|
|
# Number of calibration runs to use for calibrating the tests |
|
CALIBRATION_RUNS = 20 |
|
|
|
# Number of calibration loops to run for each calibration run |
|
CALIBRATION_LOOPS = 20 |
|
|
|
# Allow skipping calibration ? |
|
ALLOW_SKIPPING_CALIBRATION = 1 |
|
|
|
# Timer types |
|
TIMER_TIME_TIME = 'time.time' |
|
TIMER_TIME_CLOCK = 'time.clock' |
|
TIMER_SYSTIMES_PROCESSTIME = 'systimes.processtime' |
|
|
|
# Choose platform default timer |
|
if sys.platform[:3] == 'win': |
|
# On WinXP this has 2.5ms resolution |
|
TIMER_PLATFORM_DEFAULT = TIMER_TIME_CLOCK |
|
else: |
|
# On Linux this has 1ms resolution |
|
TIMER_PLATFORM_DEFAULT = TIMER_TIME_TIME |
|
|
|
# Print debug information ? |
|
_debug = 0 |
|
|
|
### Helpers |
|
|
|
def get_timer(timertype): |
|
|
|
if timertype == TIMER_TIME_TIME: |
|
return time.time |
|
elif timertype == TIMER_TIME_CLOCK: |
|
return time.clock |
|
elif timertype == TIMER_SYSTIMES_PROCESSTIME: |
|
import systimes |
|
return systimes.processtime |
|
else: |
|
raise TypeError('unknown timer type: %s' % timertype) |
|
|
|
def get_machine_details(): |
|
|
|
if _debug: |
|
print 'Getting machine details...' |
|
buildno, builddate = platform.python_build() |
|
python = platform.python_version() |
|
try: |
|
unichr(100000) |
|
except ValueError: |
|
# UCS2 build (standard) |
|
unicode = 'UCS2' |
|
except NameError: |
|
unicode = None |
|
else: |
|
# UCS4 build (most recent Linux distros) |
|
unicode = 'UCS4' |
|
bits, linkage = platform.architecture() |
|
return { |
|
'platform': platform.platform(), |
|
'processor': platform.processor(), |
|
'executable': sys.executable, |
|
'implementation': getattr(platform, 'python_implementation', |
|
lambda:'n/a')(), |
|
'python': platform.python_version(), |
|
'compiler': platform.python_compiler(), |
|
'buildno': buildno, |
|
'builddate': builddate, |
|
'unicode': unicode, |
|
'bits': bits, |
|
} |
|
|
|
def print_machine_details(d, indent=''): |
|
|
|
l = ['Machine Details:', |
|
' Platform ID: %s' % d.get('platform', 'n/a'), |
|
' Processor: %s' % d.get('processor', 'n/a'), |
|
'', |
|
'Python:', |
|
' Implementation: %s' % d.get('implementation', 'n/a'), |
|
' Executable: %s' % d.get('executable', 'n/a'), |
|
' Version: %s' % d.get('python', 'n/a'), |
|
' Compiler: %s' % d.get('compiler', 'n/a'), |
|
' Bits: %s' % d.get('bits', 'n/a'), |
|
' Build: %s (#%s)' % (d.get('builddate', 'n/a'), |
|
d.get('buildno', 'n/a')), |
|
' Unicode: %s' % d.get('unicode', 'n/a'), |
|
] |
|
print indent + string.join(l, '\n' + indent) + '\n' |
|
|
|
### Test baseclass |
|
|
|
class Test: |
|
|
|
""" All test must have this class as baseclass. It provides |
|
the necessary interface to the benchmark machinery. |
|
|
|
The tests must set .rounds to a value high enough to let the |
|
test run between 20-50 seconds. This is needed because |
|
clock()-timing only gives rather inaccurate values (on Linux, |
|
for example, it is accurate to a few hundreths of a |
|
second). If you don't want to wait that long, use a warp |
|
factor larger than 1. |
|
|
|
It is also important to set the .operations variable to a |
|
value representing the number of "virtual operations" done per |
|
call of .run(). |
|
|
|
If you change a test in some way, don't forget to increase |
|
its version number. |
|
|
|
""" |
|
|
|
### Instance variables that each test should override |
|
|
|
# Version number of the test as float (x.yy); this is important |
|
# for comparisons of benchmark runs - tests with unequal version |
|
# number will not get compared. |
|
version = 2.0 |
|
|
|
# The number of abstract operations done in each round of the |
|
# test. An operation is the basic unit of what you want to |
|
# measure. The benchmark will output the amount of run-time per |
|
# operation. Note that in order to raise the measured timings |
|
# significantly above noise level, it is often required to repeat |
|
# sets of operations more than once per test round. The measured |
|
# overhead per test round should be less than 1 second. |
|
operations = 1 |
|
|
|
# Number of rounds to execute per test run. This should be |
|
# adjusted to a figure that results in a test run-time of between |
|
# 1-2 seconds. |
|
rounds = 100000 |
|
|
|
### Internal variables |
|
|
|
# Mark this class as implementing a test |
|
is_a_test = 1 |
|
|
|
# Last timing: (real, run, overhead) |
|
last_timing = (0.0, 0.0, 0.0) |
|
|
|
# Warp factor to use for this test |
|
warp = 1 |
|
|
|
# Number of calibration runs to use |
|
calibration_runs = CALIBRATION_RUNS |
|
|
|
# List of calibration timings |
|
overhead_times = None |
|
|
|
# List of test run timings |
|
times = [] |
|
|
|
# Timer used for the benchmark |
|
timer = TIMER_PLATFORM_DEFAULT |
|
|
|
def __init__(self, warp=None, calibration_runs=None, timer=None): |
|
|
|
# Set parameters |
|
if warp is not None: |
|
self.rounds = int(self.rounds / warp) |
|
if self.rounds == 0: |
|
raise ValueError('warp factor set too high') |
|
self.warp = warp |
|
if calibration_runs is not None: |
|
if (not ALLOW_SKIPPING_CALIBRATION and |
|
calibration_runs < 1): |
|
raise ValueError('at least one calibration run is required') |
|
self.calibration_runs = calibration_runs |
|
if timer is not None: |
|
self.timer = timer |
|
|
|
# Init variables |
|
self.times = [] |
|
self.overhead_times = [] |
|
|
|
# We want these to be in the instance dict, so that pickle |
|
# saves them |
|
self.version = self.version |
|
self.operations = self.operations |
|
self.rounds = self.rounds |
|
|
|
def get_timer(self): |
|
|
|
""" Return the timer function to use for the test. |
|
|
|
""" |
|
return get_timer(self.timer) |
|
|
|
def compatible(self, other): |
|
|
|
""" Return 1/0 depending on whether the test is compatible |
|
with the other Test instance or not. |
|
|
|
""" |
|
if self.version != other.version: |
|
return 0 |
|
if self.rounds != other.rounds: |
|
return 0 |
|
return 1 |
|
|
|
def calibrate_test(self): |
|
|
|
if self.calibration_runs == 0: |
|
self.overhead_times = [0.0] |
|
return |
|
|
|
calibrate = self.calibrate |
|
timer = self.get_timer() |
|
calibration_loops = range(CALIBRATION_LOOPS) |
|
|
|
# Time the calibration loop overhead |
|
prep_times = [] |
|
for i in range(self.calibration_runs): |
|
t = timer() |
|
for i in calibration_loops: |
|
pass |
|
t = timer() - t |
|
prep_times.append(t / CALIBRATION_LOOPS) |
|
min_prep_time = min(prep_times) |
|
if _debug: |
|
print |
|
print 'Calib. prep time = %.6fms' % ( |
|
min_prep_time * MILLI_SECONDS) |
|
|
|
# Time the calibration runs (doing CALIBRATION_LOOPS loops of |
|
# .calibrate() method calls each) |
|
for i in range(self.calibration_runs): |
|
t = timer() |
|
for i in calibration_loops: |
|
calibrate() |
|
t = timer() - t |
|
self.overhead_times.append(t / CALIBRATION_LOOPS |
|
- min_prep_time) |
|
|
|
# Check the measured times |
|
min_overhead = min(self.overhead_times) |
|
max_overhead = max(self.overhead_times) |
|
if _debug: |
|
print 'Calib. overhead time = %.6fms' % ( |
|
min_overhead * MILLI_SECONDS) |
|
if min_overhead < 0.0: |
|
raise ValueError('calibration setup did not work') |
|
if max_overhead - min_overhead > 0.1: |
|
raise ValueError( |
|
'overhead calibration timing range too inaccurate: ' |
|
'%r - %r' % (min_overhead, max_overhead)) |
|
|
|
def run(self): |
|
|
|
""" Run the test in two phases: first calibrate, then |
|
do the actual test. Be careful to keep the calibration |
|
timing low w/r to the test timing. |
|
|
|
""" |
|
test = self.test |
|
timer = self.get_timer() |
|
|
|
# Get calibration |
|
min_overhead = min(self.overhead_times) |
|
|
|
# Test run |
|
t = timer() |
|
test() |
|
t = timer() - t |
|
if t < MIN_TEST_RUNTIME: |
|
raise ValueError('warp factor too high: ' |
|
'test times are < 10ms') |
|
eff_time = t - min_overhead |
|
if eff_time < 0: |
|
raise ValueError('wrong calibration') |
|
self.last_timing = (eff_time, t, min_overhead) |
|
self.times.append(eff_time) |
|
|
|
def calibrate(self): |
|
|
|
""" Calibrate the test. |
|
|
|
This method should execute everything that is needed to |
|
setup and run the test - except for the actual operations |
|
that you intend to measure. pybench uses this method to |
|
measure the test implementation overhead. |
|
|
|
""" |
|
return |
|
|
|
def test(self): |
|
|
|
""" Run the test. |
|
|
|
The test needs to run self.rounds executing |
|
self.operations number of operations each. |
|
|
|
""" |
|
return |
|
|
|
def stat(self): |
|
|
|
""" Return test run statistics as tuple: |
|
|
|
(minimum run time, |
|
average run time, |
|
total run time, |
|
average time per operation, |
|
minimum overhead time) |
|
|
|
""" |
|
runs = len(self.times) |
|
if runs == 0: |
|
return 0.0, 0.0, 0.0, 0.0 |
|
min_time = min(self.times) |
|
total_time = reduce(operator.add, self.times, 0.0) |
|
avg_time = total_time / float(runs) |
|
operation_avg = total_time / float(runs |
|
* self.rounds |
|
* self.operations) |
|
if self.overhead_times: |
|
min_overhead = min(self.overhead_times) |
|
else: |
|
min_overhead = self.last_timing[2] |
|
return min_time, avg_time, total_time, operation_avg, min_overhead |
|
|
|
### Load Setup |
|
|
|
# This has to be done after the definition of the Test class, since |
|
# the Setup module will import subclasses using this class. |
|
|
|
import Setup |
|
|
|
### Benchmark base class |
|
|
|
class Benchmark: |
|
|
|
# Name of the benchmark |
|
name = '' |
|
|
|
# Number of benchmark rounds to run |
|
rounds = 1 |
|
|
|
# Warp factor use to run the tests |
|
warp = 1 # Warp factor |
|
|
|
# Average benchmark round time |
|
roundtime = 0 |
|
|
|
# Benchmark version number as float x.yy |
|
version = 2.0 |
|
|
|
# Produce verbose output ? |
|
verbose = 0 |
|
|
|
# Dictionary with the machine details |
|
machine_details = None |
|
|
|
# Timer used for the benchmark |
|
timer = TIMER_PLATFORM_DEFAULT |
|
|
|
def __init__(self, name, verbose=None, timer=None, warp=None, |
|
calibration_runs=None): |
|
|
|
if name: |
|
self.name = name |
|
else: |
|
self.name = '%04i-%02i-%02i %02i:%02i:%02i' % \ |
|
(time.localtime(time.time())[:6]) |
|
if verbose is not None: |
|
self.verbose = verbose |
|
if timer is not None: |
|
self.timer = timer |
|
if warp is not None: |
|
self.warp = warp |
|
if calibration_runs is not None: |
|
self.calibration_runs = calibration_runs |
|
|
|
# Init vars |
|
self.tests = {} |
|
if _debug: |
|
print 'Getting machine details...' |
|
self.machine_details = get_machine_details() |
|
|
|
# Make .version an instance attribute to have it saved in the |
|
# Benchmark pickle |
|
self.version = self.version |
|
|
|
def get_timer(self): |
|
|
|
""" Return the timer function to use for the test. |
|
|
|
""" |
|
return get_timer(self.timer) |
|
|
|
def compatible(self, other): |
|
|
|
""" Return 1/0 depending on whether the benchmark is |
|
compatible with the other Benchmark instance or not. |
|
|
|
""" |
|
if self.version != other.version: |
|
return 0 |
|
if (self.machine_details == other.machine_details and |
|
self.timer != other.timer): |
|
return 0 |
|
if (self.calibration_runs == 0 and |
|
other.calibration_runs != 0): |
|
return 0 |
|
if (self.calibration_runs != 0 and |
|
other.calibration_runs == 0): |
|
return 0 |
|
return 1 |
|
|
|
def load_tests(self, setupmod, limitnames=None): |
|
|
|
# Add tests |
|
if self.verbose: |
|
print 'Searching for tests ...' |
|
print '--------------------------------------' |
|
for testclass in setupmod.__dict__.values(): |
|
if not hasattr(testclass, 'is_a_test'): |
|
continue |
|
name = testclass.__name__ |
|
if name == 'Test': |
|
continue |
|
if (limitnames is not None and |
|
limitnames.search(name) is None): |
|
continue |
|
self.tests[name] = testclass( |
|
warp=self.warp, |
|
calibration_runs=self.calibration_runs, |
|
timer=self.timer) |
|
l = self.tests.keys() |
|
l.sort() |
|
if self.verbose: |
|
for name in l: |
|
print ' %s' % name |
|
print '--------------------------------------' |
|
print ' %i tests found' % len(l) |
|
print |
|
|
|
def calibrate(self): |
|
|
|
print 'Calibrating tests. Please wait...', |
|
sys.stdout.flush() |
|
if self.verbose: |
|
print |
|
print |
|
print 'Test min max' |
|
print '-' * LINE |
|
tests = self.tests.items() |
|
tests.sort() |
|
for i in range(len(tests)): |
|
name, test = tests[i] |
|
test.calibrate_test() |
|
if self.verbose: |
|
print '%30s: %6.3fms %6.3fms' % \ |
|
(name, |
|
min(test.overhead_times) * MILLI_SECONDS, |
|
max(test.overhead_times) * MILLI_SECONDS) |
|
if self.verbose: |
|
print |
|
print 'Done with the calibration.' |
|
else: |
|
print 'done.' |
|
print |
|
|
|
def run(self): |
|
|
|
tests = self.tests.items() |
|
tests.sort() |
|
timer = self.get_timer() |
|
print 'Running %i round(s) of the suite at warp factor %i:' % \ |
|
(self.rounds, self.warp) |
|
print |
|
self.roundtimes = [] |
|
for i in range(self.rounds): |
|
if self.verbose: |
|
print ' Round %-25i effective absolute overhead' % (i+1) |
|
total_eff_time = 0.0 |
|
for j in range(len(tests)): |
|
name, test = tests[j] |
|
if self.verbose: |
|
print '%30s:' % name, |
|
test.run() |
|
(eff_time, abs_time, min_overhead) = test.last_timing |
|
total_eff_time = total_eff_time + eff_time |
|
if self.verbose: |
|
print ' %5.0fms %5.0fms %7.3fms' % \ |
|
(eff_time * MILLI_SECONDS, |
|
abs_time * MILLI_SECONDS, |
|
min_overhead * MILLI_SECONDS) |
|
self.roundtimes.append(total_eff_time) |
|
if self.verbose: |
|
print (' ' |
|
' ------------------------------') |
|
print (' ' |
|
' Totals: %6.0fms' % |
|
(total_eff_time * MILLI_SECONDS)) |
|
print |
|
else: |
|
print '* Round %i done in %.3f seconds.' % (i+1, |
|
total_eff_time) |
|
print |
|
|
|
def stat(self): |
|
|
|
""" Return benchmark run statistics as tuple: |
|
|
|
(minimum round time, |
|
average round time, |
|
maximum round time) |
|
|
|
XXX Currently not used, since the benchmark does test |
|
statistics across all rounds. |
|
|
|
""" |
|
runs = len(self.roundtimes) |
|
if runs == 0: |
|
return 0.0, 0.0 |
|
min_time = min(self.roundtimes) |
|
total_time = reduce(operator.add, self.roundtimes, 0.0) |
|
avg_time = total_time / float(runs) |
|
max_time = max(self.roundtimes) |
|
return (min_time, avg_time, max_time) |
|
|
|
def print_header(self, title='Benchmark'): |
|
|
|
print '-' * LINE |
|
print '%s: %s' % (title, self.name) |
|
print '-' * LINE |
|
print |
|
print ' Rounds: %s' % self.rounds |
|
print ' Warp: %s' % self.warp |
|
print ' Timer: %s' % self.timer |
|
print |
|
if self.machine_details: |
|
print_machine_details(self.machine_details, indent=' ') |
|
print |
|
|
|
def print_benchmark(self, hidenoise=0, limitnames=None): |
|
|
|
print ('Test ' |
|
' minimum average operation overhead') |
|
print '-' * LINE |
|
tests = self.tests.items() |
|
tests.sort() |
|
total_min_time = 0.0 |
|
total_avg_time = 0.0 |
|
for name, test in tests: |
|
if (limitnames is not None and |
|
limitnames.search(name) is None): |
|
continue |
|
(min_time, |
|
avg_time, |
|
total_time, |
|
op_avg, |
|
min_overhead) = test.stat() |
|
total_min_time = total_min_time + min_time |
|
total_avg_time = total_avg_time + avg_time |
|
print '%30s: %5.0fms %5.0fms %6.2fus %7.3fms' % \ |
|
(name, |
|
min_time * MILLI_SECONDS, |
|
avg_time * MILLI_SECONDS, |
|
op_avg * MICRO_SECONDS, |
|
min_overhead *MILLI_SECONDS) |
|
print '-' * LINE |
|
print ('Totals: ' |
|
' %6.0fms %6.0fms' % |
|
(total_min_time * MILLI_SECONDS, |
|
total_avg_time * MILLI_SECONDS, |
|
)) |
|
print |
|
|
|
def print_comparison(self, compare_to, hidenoise=0, limitnames=None): |
|
|
|
# Check benchmark versions |
|
if compare_to.version != self.version: |
|
print ('* Benchmark versions differ: ' |
|
'cannot compare this benchmark to "%s" !' % |
|
compare_to.name) |
|
print |
|
self.print_benchmark(hidenoise=hidenoise, |
|
limitnames=limitnames) |
|
return |
|
|
|
# Print header |
|
compare_to.print_header('Comparing with') |
|
print ('Test ' |
|
' minimum run-time average run-time') |
|
print (' ' |
|
' this other diff this other diff') |
|
print '-' * LINE |
|
|
|
# Print test comparisons |
|
tests = self.tests.items() |
|
tests.sort() |
|
total_min_time = other_total_min_time = 0.0 |
|
total_avg_time = other_total_avg_time = 0.0 |
|
benchmarks_compatible = self.compatible(compare_to) |
|
tests_compatible = 1 |
|
for name, test in tests: |
|
if (limitnames is not None and |
|
limitnames.search(name) is None): |
|
continue |
|
(min_time, |
|
avg_time, |
|
total_time, |
|
op_avg, |
|
min_overhead) = test.stat() |
|
total_min_time = total_min_time + min_time |
|
total_avg_time = total_avg_time + avg_time |
|
try: |
|
other = compare_to.tests[name] |
|
except KeyError: |
|
other = None |
|
if other is None: |
|
# Other benchmark doesn't include the given test |
|
min_diff, avg_diff = 'n/a', 'n/a' |
|
other_min_time = 0.0 |
|
other_avg_time = 0.0 |
|
tests_compatible = 0 |
|
else: |
|
(other_min_time, |
|
other_avg_time, |
|
other_total_time, |
|
other_op_avg, |
|
other_min_overhead) = other.stat() |
|
other_total_min_time = other_total_min_time + other_min_time |
|
other_total_avg_time = other_total_avg_time + other_avg_time |
|
if (benchmarks_compatible and |
|
test.compatible(other)): |
|
# Both benchmark and tests are comparable |
|
min_diff = ((min_time * self.warp) / |
|
(other_min_time * other.warp) - 1.0) |
|
avg_diff = ((avg_time * self.warp) / |
|
(other_avg_time * other.warp) - 1.0) |
|
if hidenoise and abs(min_diff) < 10.0: |
|
min_diff = '' |
|
else: |
|
min_diff = '%+5.1f%%' % (min_diff * PERCENT) |
|
if hidenoise and abs(avg_diff) < 10.0: |
|
avg_diff = '' |
|
else: |
|
avg_diff = '%+5.1f%%' % (avg_diff * PERCENT) |
|
else: |
|
# Benchmark or tests are not comparable |
|
min_diff, avg_diff = 'n/a', 'n/a' |
|
tests_compatible = 0 |
|
print '%30s: %5.0fms %5.0fms %7s %5.0fms %5.0fms %7s' % \ |
|
(name, |
|
min_time * MILLI_SECONDS, |
|
other_min_time * MILLI_SECONDS * compare_to.warp / self.warp, |
|
min_diff, |
|
avg_time * MILLI_SECONDS, |
|
other_avg_time * MILLI_SECONDS * compare_to.warp / self.warp, |
|
avg_diff) |
|
print '-' * LINE |
|
|
|
# Summarise test results |
|
if not benchmarks_compatible or not tests_compatible: |
|
min_diff, avg_diff = 'n/a', 'n/a' |
|
else: |
|
if other_total_min_time != 0.0: |
|
min_diff = '%+5.1f%%' % ( |
|
((total_min_time * self.warp) / |
|
(other_total_min_time * compare_to.warp) - 1.0) * PERCENT) |
|
else: |
|
min_diff = 'n/a' |
|
if other_total_avg_time != 0.0: |
|
avg_diff = '%+5.1f%%' % ( |
|
((total_avg_time * self.warp) / |
|
(other_total_avg_time * compare_to.warp) - 1.0) * PERCENT) |
|
else: |
|
avg_diff = 'n/a' |
|
print ('Totals: ' |
|
' %5.0fms %5.0fms %7s %5.0fms %5.0fms %7s' % |
|
(total_min_time * MILLI_SECONDS, |
|
(other_total_min_time * compare_to.warp/self.warp |
|
* MILLI_SECONDS), |
|
min_diff, |
|
total_avg_time * MILLI_SECONDS, |
|
(other_total_avg_time * compare_to.warp/self.warp |
|
* MILLI_SECONDS), |
|
avg_diff |
|
)) |
|
print |
|
print '(this=%s, other=%s)' % (self.name, |
|
compare_to.name) |
|
print |
|
|
|
class PyBenchCmdline(Application): |
|
|
|
header = ("PYBENCH - a benchmark test suite for Python " |
|
"interpreters/compilers.") |
|
|
|
version = __version__ |
|
|
|
debug = _debug |
|
|
|
options = [ArgumentOption('-n', |
|
'number of rounds', |
|
Setup.Number_of_rounds), |
|
ArgumentOption('-f', |
|
'save benchmark to file arg', |
|
''), |
|
ArgumentOption('-c', |
|
'compare benchmark with the one in file arg', |
|
''), |
|
ArgumentOption('-s', |
|
'show benchmark in file arg, then exit', |
|
''), |
|
ArgumentOption('-w', |
|
'set warp factor to arg', |
|
Setup.Warp_factor), |
|
ArgumentOption('-t', |
|
'run only tests with names matching arg', |
|
''), |
|
ArgumentOption('-C', |
|
'set the number of calibration runs to arg', |
|
CALIBRATION_RUNS), |
|
SwitchOption('-d', |
|
'hide noise in comparisons', |
|
0), |
|
SwitchOption('-v', |
|
'verbose output (not recommended)', |
|
0), |
|
SwitchOption('--with-gc', |
|
'enable garbage collection', |
|
0), |
|
SwitchOption('--with-syscheck', |
|
'use default sys check interval', |
|
0), |
|
ArgumentOption('--timer', |
|
'use given timer', |
|
TIMER_PLATFORM_DEFAULT), |
|
] |
|
|
|
about = """\ |
|
The normal operation is to run the suite and display the |
|
results. Use -f to save them for later reuse or comparisons. |
|
|
|
Available timers: |
|
|
|
time.time |
|
time.clock |
|
systimes.processtime |
|
|
|
Examples: |
|
|
|
python2.1 pybench.py -f p21.pybench |
|
python2.5 pybench.py -f p25.pybench |
|
python pybench.py -s p25.pybench -c p21.pybench |
|
""" |
|
copyright = __copyright__ |
|
|
|
def main(self): |
|
|
|
rounds = self.values['-n'] |
|
reportfile = self.values['-f'] |
|
show_bench = self.values['-s'] |
|
compare_to = self.values['-c'] |
|
hidenoise = self.values['-d'] |
|
warp = int(self.values['-w']) |
|
withgc = self.values['--with-gc'] |
|
limitnames = self.values['-t'] |
|
if limitnames: |
|
if _debug: |
|
print '* limiting test names to one with substring "%s"' % \ |
|
limitnames |
|
limitnames = re.compile(limitnames, re.I) |
|
else: |
|
limitnames = None |
|
verbose = self.verbose |
|
withsyscheck = self.values['--with-syscheck'] |
|
calibration_runs = self.values['-C'] |
|
timer = self.values['--timer'] |
|
|
|
print '-' * LINE |
|
print 'PYBENCH %s' % __version__ |
|
print '-' * LINE |
|
print '* using %s %s' % ( |
|
getattr(platform, 'python_implementation', lambda:'Python')(), |
|
string.join(string.split(sys.version), ' ')) |
|
|
|
# Switch off garbage collection |
|
if not withgc: |
|
try: |
|
import gc |
|
except ImportError: |
|
print '* Python version doesn\'t support garbage collection' |
|
else: |
|
try: |
|
gc.disable() |
|
except NotImplementedError: |
|
print '* Python version doesn\'t support gc.disable' |
|
else: |
|
print '* disabled garbage collection' |
|
|
|
# "Disable" sys check interval |
|
if not withsyscheck: |
|
# Too bad the check interval uses an int instead of a long... |
|
value = 2147483647 |
|
try: |
|
sys.setcheckinterval(value) |
|
except (AttributeError, NotImplementedError): |
|
print '* Python version doesn\'t support sys.setcheckinterval' |
|
else: |
|
print '* system check interval set to maximum: %s' % value |
|
|
|
if timer == TIMER_SYSTIMES_PROCESSTIME: |
|
import systimes |
|
print '* using timer: systimes.processtime (%s)' % \ |
|
systimes.SYSTIMES_IMPLEMENTATION |
|
else: |
|
print '* using timer: %s' % timer |
|
|
|
print |
|
|
|
if compare_to: |
|
try: |
|
f = open(compare_to,'rb') |
|
bench = pickle.load(f) |
|
bench.name = compare_to |
|
f.close() |
|
compare_to = bench |
|
except IOError, reason: |
|
print '* Error opening/reading file %s: %s' % ( |
|
repr(compare_to), |
|
reason) |
|
compare_to = None |
|
|
|
if show_bench: |
|
try: |
|
f = open(show_bench,'rb') |
|
bench = pickle.load(f) |
|
bench.name = show_bench |
|
f.close() |
|
bench.print_header() |
|
if compare_to: |
|
bench.print_comparison(compare_to, |
|
hidenoise=hidenoise, |
|
limitnames=limitnames) |
|
else: |
|
bench.print_benchmark(hidenoise=hidenoise, |
|
limitnames=limitnames) |
|
except IOError, reason: |
|
print '* Error opening/reading file %s: %s' % ( |
|
repr(show_bench), |
|
reason) |
|
print |
|
return |
|
|
|
if reportfile: |
|
print 'Creating benchmark: %s (rounds=%i, warp=%i)' % \ |
|
(reportfile, rounds, warp) |
|
print |
|
|
|
# Create benchmark object |
|
bench = Benchmark(reportfile, |
|
verbose=verbose, |
|
timer=timer, |
|
warp=warp, |
|
calibration_runs=calibration_runs) |
|
bench.rounds = rounds |
|
bench.load_tests(Setup, limitnames=limitnames) |
|
try: |
|
bench.calibrate() |
|
bench.run() |
|
except KeyboardInterrupt: |
|
print |
|
print '*** KeyboardInterrupt -- Aborting' |
|
print |
|
return |
|
bench.print_header() |
|
if compare_to: |
|
bench.print_comparison(compare_to, |
|
hidenoise=hidenoise, |
|
limitnames=limitnames) |
|
else: |
|
bench.print_benchmark(hidenoise=hidenoise, |
|
limitnames=limitnames) |
|
|
|
# Ring bell |
|
sys.stderr.write('\007') |
|
|
|
if reportfile: |
|
try: |
|
f = open(reportfile,'wb') |
|
bench.name = reportfile |
|
pickle.dump(bench,f) |
|
f.close() |
|
except IOError, reason: |
|
print '* Error opening/writing reportfile' |
|
except IOError, reason: |
|
print '* Error opening/writing reportfile %s: %s' % ( |
|
reportfile, |
|
reason) |
|
print |
|
|
|
if __name__ == '__main__': |
|
PyBenchCmdline() |
