| #!/usr/bin/python3 |
| # |
| # Copyright 2018, The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """MLTS benchmark result generator. |
| |
| Reads a CSV produced by MLTS benchmark and generates |
| an HTML page with results summary. |
| |
| Usage: |
| generate_result [csv input file] [html output file] |
| """ |
| |
| import argparse |
| import collections |
| import csv |
| import os |
| import re |
| |
| |
| class ScoreException(Exception): |
| """Generator base exception type. """ |
| pass |
| |
| |
| BenchmarkResult = collections.namedtuple( |
| 'BenchmarkResult', |
| ['name', 'backend_type', 'iterations', 'total_time_sec', 'max_single_error', |
| 'testset_size', 'evaluator_keys', 'evaluator_values', |
| 'time_freq_start_sec', 'time_freq_step_sec', 'time_freq_sec', |
| 'validation_errors']) |
| |
| |
| ResultsWithBaseline = collections.namedtuple( |
| 'ResultsWithBaseline', |
| ['baseline', 'other']) |
| |
| |
| BASELINE_BACKEND = 'TFLite_CPU' |
| KNOWN_GROUPS = [ |
| (re.compile('mobilenet_v1.*quant.*'), 'MobileNet v1 Quantized'), |
| (re.compile('mobilenet_v1.*'), 'MobileNet v1 Float'), |
| (re.compile('mobilenet_v2.*quant.*'), 'MobileNet v2 Quantized'), |
| (re.compile('mobilenet_v2.*'), 'MobileNet v2 Float'), |
| (re.compile('tts.*'), 'LSTM Text-to-speech'), |
| (re.compile('asr.*'), 'LSTM Automatic Speech Recognition'), |
| ] |
| |
| |
| def parse_csv_input(input_filename): |
| """Parse input CSV file, returns: (benchmarkInfo, list of BenchmarkResult).""" |
| with open(input_filename, 'r') as csvfile: |
| csv_reader = csv.reader(filter(lambda row: row[0] != '#', csvfile)) |
| |
| # First line contain device info |
| benchmark_info = next(csv_reader) |
| |
| results = [] |
| for row in csv_reader: |
| evaluator_keys_count = int(row[8]) |
| time_freq_sec_count = int(row[9]) |
| validation_error_count = int(row[10]) |
| |
| tf_start = 11 + evaluator_keys_count*2 |
| time_freq_sec = [float(x) for x in |
| row[tf_start:tf_start + time_freq_sec_count]] |
| ve_start = 11 + evaluator_keys_count*2 + time_freq_sec_count |
| validation_errors = row[ve_start: ve_start + validation_error_count] |
| |
| results.append(BenchmarkResult( |
| name=row[0], |
| backend_type=row[1], |
| iterations=int(row[2]), |
| total_time_sec=float(row[3]), |
| max_single_error=float(row[4]), |
| testset_size=int(row[5]), |
| time_freq_start_sec=float(row[6]), |
| time_freq_step_sec=float(row[7]), |
| evaluator_keys=row[11:11 + evaluator_keys_count], |
| evaluator_values=row[ |
| 11 + evaluator_keys_count: 11 + evaluator_keys_count*2], |
| time_freq_sec=time_freq_sec, |
| validation_errors=validation_errors, |
| )) |
| return (benchmark_info, results) |
| |
| |
| def group_results(results): |
| """Group list of results by their name/backend, returns list of lists.""" |
| # Group by name |
| groupings = collections.defaultdict(list) |
| for result in results: |
| groupings[result.name].append(result) |
| |
| # Find baseline for each group, make ResultsWithBaseline for each name |
| groupings_baseline = {} |
| for name, results in groupings.items(): |
| baseline = next(filter(lambda x: x.backend_type == BASELINE_BACKEND, |
| results)) |
| other = sorted(filter(lambda x: x is not baseline, results), |
| key=lambda x: x.backend_type) |
| groupings_baseline[name] = ResultsWithBaseline( |
| baseline=baseline, |
| other=other) |
| |
| # Merge ResultsWithBaseline for known groups |
| known_groupings_baseline = collections.defaultdict(list) |
| for name, results_with_bl in sorted(groupings_baseline.items()): |
| group_name = name |
| for known_group in KNOWN_GROUPS: |
| if known_group[0].match(results_with_bl.baseline.name): |
| group_name = known_group[1] |
| break |
| known_groupings_baseline[group_name].append(results_with_bl) |
| |
| # Turn into a list sorted by name |
| groupings_list = [] |
| for name, results_wbl in sorted(known_groupings_baseline.items()): |
| groupings_list.append((name, results_wbl)) |
| return groupings_list |
| |
| |
| def get_frequency_graph_min_max(results_with_bl): |
| """Get min and max times of latencies frequency.""" |
| mins = [] |
| maxs = [] |
| for result in [results_with_bl.baseline] + results_with_bl.other: |
| mins.append(result.time_freq_start_sec) |
| to_add = len(result.time_freq_sec) * result.time_freq_step_sec |
| maxs.append(result.time_freq_start_sec + to_add) |
| return min(mins), max(maxs) |
| |
| |
| def get_frequency_graph(time_freq_start_sec, time_freq_step_sec, time_freq_sec, |
| start_sec, end_sec): |
| """Generate input x/y data for latency frequency graph.""" |
| left_to_pad = int((time_freq_start_sec - start_sec) / time_freq_step_sec) |
| end_time = time_freq_start_sec + len(time_freq_sec) * time_freq_step_sec |
| right_to_pad = int((end_sec - end_time) / time_freq_step_sec) |
| |
| # After pading more that 64 values, graphs start to look messy, |
| # bail out in that case. |
| if (left_to_pad + right_to_pad) < 64: |
| left_pad = (['{:.2f}ms'.format( |
| (start_sec + x * time_freq_step_sec) * 1000.0) |
| for x in range(left_to_pad)], [0] * left_to_pad) |
| |
| right_pad = (['{:.2f}ms'.format( |
| (end_time + x * time_freq_step_sec) * 1000.0) |
| for x in range(right_to_pad)], [0] * right_to_pad) |
| else: |
| left_pad = [[], []] |
| right_pad = [[], []] |
| |
| data = (['{:.2f}ms'.format( |
| (time_freq_start_sec + x * time_freq_step_sec) * 1000.0) |
| for x in range(len(time_freq_sec))], time_freq_sec) |
| |
| return (left_pad[0] + data[0] + right_pad[0], |
| left_pad[1] + data[1] + right_pad[1]) |
| |
| |
| def is_topk_evaluator(evaluator_keys): |
| """Are these evaluator keys from TopK evaluator?""" |
| return (len(evaluator_keys) == 5 and |
| evaluator_keys[0] == 'top_1' and |
| evaluator_keys[1] == 'top_2' and |
| evaluator_keys[2] == 'top_3' and |
| evaluator_keys[3] == 'top_4' and |
| evaluator_keys[4] == 'top_5') |
| |
| |
| def is_melceplogf0_evaluator(evaluator_keys): |
| """Are these evaluator keys from MelCepLogF0 evaluator?""" |
| return (len(evaluator_keys) == 2 and |
| evaluator_keys[0] == 'max_mel_cep_distortion' and |
| evaluator_keys[1] == 'max_log_f0_error') |
| |
| |
| def is_phone_error_rate_evaluator(evaluator_keys): |
| """Are these evaluator keys from PhoneErrorRate evaluator?""" |
| return (len(evaluator_keys) == 1 and |
| evaluator_keys[0] == 'max_phone_error_rate') |
| |
| |
| def generate_accuracy_headers(result): |
| """Accuracy-related headers for result table.""" |
| if is_topk_evaluator(result.evaluator_keys): |
| return ACCURACY_HEADERS_TOPK_TEMPLATE |
| elif is_melceplogf0_evaluator(result.evaluator_keys): |
| return ACCURACY_HEADERS_MELCEPLOGF0_TEMPLATE |
| elif is_phone_error_rate_evaluator(result.evaluator_keys): |
| return ACCURACY_HEADERS_PHONE_ERROR_RATE_TEMPLATE |
| else: |
| return ACCURACY_HEADERS_BASIC_TEMPLATE |
| raise ScoreException('Unknown accuracy headers for: ' + str(result)) |
| |
| |
| def get_diff_span(value, same_delta, positive_is_better): |
| if abs(value) < same_delta: |
| return 'same' |
| if positive_is_better and value > 0 or not positive_is_better and value < 0: |
| return 'better' |
| return 'worse' |
| |
| |
| def generate_accuracy_values(baseline, result): |
| """Accuracy-related data for result table.""" |
| if is_topk_evaluator(result.evaluator_keys): |
| val = [float(x) * 100.0 for x in result.evaluator_values] |
| if result is baseline: |
| topk = [TOPK_BASELINE_TEMPLATE.format(val=x) for x in val] |
| return ACCURACY_VALUES_TOPK_TEMPLATE.format( |
| top1=topk[0], top2=topk[1], top3=topk[2], top4=topk[3], |
| top5=topk[4] |
| ) |
| else: |
| base = [float(x) * 100.0 for x in baseline.evaluator_values] |
| diff = [a - b for a, b in zip(val, base)] |
| topk = [TOPK_DIFF_TEMPLATE.format( |
| val=v, diff=d, span=get_diff_span(d, 1.0, positive_is_better=True)) |
| for v, d in zip(val, diff)] |
| return ACCURACY_VALUES_TOPK_TEMPLATE.format( |
| top1=topk[0], top2=topk[1], top3=topk[2], top4=topk[3], |
| top5=topk[4] |
| ) |
| elif is_melceplogf0_evaluator(result.evaluator_keys): |
| val = [float(x) for x in |
| result.evaluator_values + [result.max_single_error]] |
| if result is baseline: |
| return ACCURACY_VALUES_MELCEPLOGF0_TEMPLATE.format( |
| max_log_f0=MELCEPLOGF0_BASELINE_TEMPLATE.format( |
| val=val[0]), |
| max_mel_cep_distortion=MELCEPLOGF0_BASELINE_TEMPLATE.format( |
| val=val[1]), |
| max_single_error=MELCEPLOGF0_BASELINE_TEMPLATE.format( |
| val=val[2]), |
| ) |
| else: |
| base = [float(x) for x in |
| baseline.evaluator_values + [baseline.max_single_error]] |
| diff = [a - b for a, b in zip(val, base)] |
| v = [MELCEPLOGF0_DIFF_TEMPLATE.format( |
| val=v, diff=d, span=get_diff_span(d, 1.0, positive_is_better=False)) |
| for v, d in zip(val, diff)] |
| return ACCURACY_VALUES_MELCEPLOGF0_TEMPLATE.format( |
| max_log_f0=v[0], |
| max_mel_cep_distortion=v[1], |
| max_single_error=v[2], |
| ) |
| elif is_phone_error_rate_evaluator(result.evaluator_keys): |
| val = [float(x) for x in |
| result.evaluator_values + [result.max_single_error]] |
| if result is baseline: |
| return ACCURACY_VALUES_PHONE_ERROR_RATE_TEMPLATE.format( |
| max_phone_error_rate=PHONE_ERROR_RATE_BASELINE_TEMPLATE.format( |
| val=val[0]), |
| max_single_error=PHONE_ERROR_RATE_BASELINE_TEMPLATE.format( |
| val=val[1]), |
| ) |
| else: |
| base = [float(x) for x in |
| baseline.evaluator_values + [baseline.max_single_error]] |
| diff = [a - b for a, b in zip(val, base)] |
| v = [PHONE_ERROR_RATE_DIFF_TEMPLATE.format( |
| val=v, diff=d, span=get_diff_span(d, 1.0, positive_is_better=False)) |
| for v, d in zip(val, diff)] |
| return ACCURACY_VALUES_PHONE_ERROR_RATE_TEMPLATE.format( |
| max_phone_error_rate=v[0], |
| max_single_error=v[1], |
| ) |
| else: |
| return ACCURACY_VALUES_BASIC_TEMPLATE.format( |
| max_single_error=result.max_single_error, |
| ) |
| raise ScoreException('Unknown accuracy values for: ' + str(result)) |
| |
| |
| def getchartjs_source(): |
| return open(os.path.dirname(os.path.abspath(__file__)) + '/' + |
| CHART_JS_FILE).read() |
| |
| |
| def generate_avg_ms(baseline, result): |
| """Generate average latency value.""" |
| if result is None: |
| result = baseline |
| |
| result_avg_ms = (result.total_time_sec / result.iterations)*1000.0 |
| if result is baseline: |
| return LATENCY_BASELINE_TEMPLATE.format(val=result_avg_ms) |
| baseline_avg_ms = (baseline.total_time_sec / baseline.iterations)*1000.0 |
| diff = (result_avg_ms/baseline_avg_ms - 1.0) * 100.0 |
| diff_val = result_avg_ms - baseline_avg_ms |
| return LATENCY_DIFF_TEMPLATE.format( |
| val=result_avg_ms, |
| diff=diff, |
| diff_val=diff_val, |
| span=get_diff_span(diff, same_delta=1.0, positive_is_better=False)) |
| |
| |
| def generate_result_entry(baseline, result): |
| if result is None: |
| result = baseline |
| |
| return RESULT_ENTRY_TEMPLATE.format( |
| row_class='failed' if result.validation_errors else 'normal', |
| name=result.name, |
| backend=result.backend_type, |
| iterations=result.iterations, |
| testset_size=result.testset_size, |
| accuracy_values=generate_accuracy_values(baseline, result), |
| avg_ms=generate_avg_ms(baseline, result)) |
| |
| |
| def generate_latency_graph_entry(result, results_with_bl): |
| tmin, tmax = get_frequency_graph_min_max(results_with_bl) |
| return LATENCY_GRAPH_ENTRY_TEMPLATE.format( |
| backend=result.backend_type, |
| i=id(result), |
| freq_data=get_frequency_graph(result.time_freq_start_sec, |
| result.time_freq_step_sec, |
| result.time_freq_sec, |
| tmin, tmax)) |
| |
| |
| def generate_validation_errors(entries_group): |
| """Generate validation errors table.""" |
| errors = [] |
| for result_and_bl in entries_group: |
| for result in [result_and_bl.baseline] + result_and_bl.other: |
| for error in result.validation_errors: |
| errors.append((result.name, result.backend_type, error)) |
| |
| if errors: |
| return VALIDATION_ERRORS_TEMPLATE.format( |
| results=''.join( |
| VALIDATION_ERRORS_ENTRY_TEMPLATE.format( |
| name=name, |
| backend=backend, |
| error=error) for name, backend, error in errors)) |
| return '' |
| |
| |
| def generate_result(benchmark_info, data): |
| """Turn list of results into HTML.""" |
| return MAIN_TEMPLATE.format( |
| jsdeps=getchartjs_source(), |
| device_info=DEVICE_INFO_TEMPLATE.format( |
| benchmark_time=benchmark_info[0], |
| device_info=benchmark_info[1], |
| ), |
| results_list=''.join(( |
| RESULT_GROUP_TEMPLATE.format( |
| group_name=entries_name, |
| accuracy_headers=generate_accuracy_headers( |
| entries_group[0].baseline), |
| results=''.join( |
| RESULT_ENTRY_WITH_BASELINE_TEMPLATE.format( |
| baseline=generate_result_entry( |
| result_and_bl.baseline, None), |
| other=''.join( |
| generate_result_entry( |
| result_and_bl.baseline, x) |
| for x in result_and_bl.other) |
| ) for result_and_bl in entries_group), |
| validation_errors=generate_validation_errors(entries_group), |
| latency_graphs=LATENCY_GRAPHS_TEMPLATE.format( |
| results=''.join( |
| LATENCY_GRAPH_ENTRY_WITH_BL_TEMPLATE.format( |
| name=result_and_bl.baseline.name, |
| baseline=generate_latency_graph_entry( |
| result_and_bl.baseline, result_and_bl), |
| result=''.join( |
| generate_latency_graph_entry(x, result_and_bl) |
| for x in result_and_bl.other) |
| ) for result_and_bl in entries_group) |
| ) |
| ) for entries_name, entries_group in group_results(data)) |
| )) |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument('input', help='input csv filename') |
| parser.add_argument('output', help='output html filename') |
| args = parser.parse_args() |
| |
| benchmark_info, data = parse_csv_input(args.input) |
| |
| with open(args.output, 'w') as htmlfile: |
| htmlfile.write(generate_result(benchmark_info, data)) |
| |
| |
| # ----------------- |
| # Templates below |
| |
| MAIN_TEMPLATE = """<!doctype html> |
| <html lang='en-US'> |
| <head> |
| <meta http-equiv='Content-Type' content='text/html; charset=utf-8'> |
| <script src='https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js'></script> |
| <script>{jsdeps}</script> |
| <title>MLTS results</title> |
| <style> |
| .results {{ |
| border-collapse: collapse; |
| width: 100%; |
| }} |
| .results td, .results th {{ |
| border: 1px solid #ddd; |
| padding: 6px; |
| }} |
| .results tbody.values {{ |
| border-bottom: 8px solid #333; |
| }} |
| span.better {{ |
| color: #070; |
| }} |
| span.worse {{ |
| color: #700; |
| }} |
| span.same {{ |
| color: #000; |
| }} |
| .results tr:nth-child(even) {{background-color: #eee;}} |
| .results tr:hover {{background-color: #ddd;}} |
| .results th {{ |
| padding: 10px; |
| font-weight: bold; |
| text-align: left; |
| background-color: #333; |
| color: white; |
| }} |
| .results tr.failed {{ |
| background-color: #ffc4ca; |
| }} |
| .group {{ |
| padding-top: 25px; |
| }} |
| .group_name {{ |
| padding-left: 10px; |
| font-size: 140%; |
| font-weight: bold; |
| }} |
| .latency_results {{ |
| padding: 10px; |
| border: 1px solid #ddd; |
| overflow: hidden; |
| }} |
| .latency_with_baseline {{ |
| padding: 10px; |
| border: 1px solid #ddd; |
| overflow: hidden; |
| }} |
| </style> |
| </head> |
| <body> |
| {device_info} |
| {results_list} |
| </body> |
| </html>""" |
| |
| DEVICE_INFO_TEMPLATE = """<div id='device_info'> |
| Benchmark for {device_info}, started at {benchmark_time} |
| </div>""" |
| |
| |
| RESULT_GROUP_TEMPLATE = """<div class="group"> |
| <div class="group_name">{group_name}</div> |
| <table class="results"> |
| <tr> |
| <th>Name</th> |
| <th>Backend</th> |
| <th>Iterations</th> |
| <th>Test set size</th> |
| <th>Average latency ms</th> |
| {accuracy_headers} |
| </tr> |
| {results} |
| </table> |
| {validation_errors} |
| {latency_graphs} |
| </div>""" |
| |
| |
| VALIDATION_ERRORS_TEMPLATE = """ |
| <table class="results"> |
| <tr> |
| <th>Name</th> |
| <th>Backend</th> |
| <th>Error</th> |
| </tr> |
| {results} |
| </table>""" |
| VALIDATION_ERRORS_ENTRY_TEMPLATE = """ |
| <tr class="failed"> |
| <td>{name}</td> |
| <td>{backend}</td> |
| <td>{error}</td> |
| </tr> |
| """ |
| |
| LATENCY_GRAPHS_TEMPLATE = """ |
| <div class="latency_results"> |
| {results} |
| </div> |
| <div style="clear: left;"></div> |
| """ |
| |
| LATENCY_GRAPH_ENTRY_WITH_BL_TEMPLATE = """ |
| <div class="latency_with_baseline" style="float: left;"> |
| <b>{name}</b> |
| {baseline} |
| {result} |
| </div> |
| """ |
| |
| LATENCY_GRAPH_ENTRY_TEMPLATE = """ |
| <div class="latency_result" style='width: 350px;'> |
| {backend} |
| <canvas id='latency_chart{i}' class='latency_chart'></canvas> |
| <script> |
| $(function() {{ |
| var freqData = {{ |
| labels: {freq_data[0]}, |
| datasets: [{{ |
| data: {freq_data[1]}, |
| backgroundColor: 'rgba(255, 99, 132, 0.6)', |
| borderColor: 'rgba(255, 0, 0, 0.6)', |
| borderWidth: 1, |
| }}] |
| }}; |
| var ctx = $('#latency_chart{i}')[0].getContext('2d'); |
| window.latency_chart{i} = new Chart(ctx, |
| {{ |
| type: 'bar', |
| data: freqData, |
| options: {{ |
| responsive: true, |
| title: {{ |
| display: false, |
| text: 'Latency frequency' |
| }}, |
| legend: {{ |
| display: false |
| }}, |
| scales: {{ |
| xAxes: [ {{ |
| barPercentage: 1.0, |
| categoryPercentage: 0.9, |
| }}], |
| yAxes: [{{ |
| scaleLabel: {{ |
| display: false, |
| labelString: 'Iterations Count' |
| }} |
| }}] |
| }} |
| }} |
| }}); |
| }}); |
| </script> |
| </div> |
| """ |
| |
| |
| RESULT_ENTRY_WITH_BASELINE_TEMPLATE = """ |
| <tbody class="values"> |
| {baseline} |
| {other} |
| </tbody> |
| """ |
| RESULT_ENTRY_TEMPLATE = """ |
| <tr class={row_class}> |
| <td>{name}</td> |
| <td>{backend}</td> |
| <td>{iterations:d}</td> |
| <td>{testset_size:d}</td> |
| <td>{avg_ms}</td> |
| {accuracy_values} |
| </tr>""" |
| |
| LATENCY_BASELINE_TEMPLATE = """{val:.2f}ms""" |
| LATENCY_DIFF_TEMPLATE = """{val:.2f}ms <span class='{span}'> |
| ({diff_val:.2f}ms, {diff:.1f}%)</span>""" |
| |
| |
| ACCURACY_HEADERS_TOPK_TEMPLATE = """ |
| <th>Top 1</th> |
| <th>Top 2</th> |
| <th>Top 3</th> |
| <th>Top 4</th> |
| <th>Top 5</th> |
| """ |
| ACCURACY_VALUES_TOPK_TEMPLATE = """ |
| <td>{top1}</td> |
| <td>{top2}</td> |
| <td>{top3}</td> |
| <td>{top4}</td> |
| <td>{top5}</td> |
| """ |
| TOPK_BASELINE_TEMPLATE = """{val:.3f}%""" |
| TOPK_DIFF_TEMPLATE = """{val:.3f}% <span class='{span}'>({diff:.1f}%)</span>""" |
| |
| |
| ACCURACY_HEADERS_MELCEPLOGF0_TEMPLATE = """ |
| <th>Max log(F0) error</th> |
| <th>Max Mel Cep distortion</th> |
| <th>Max scalar error</th> |
| """ |
| |
| ACCURACY_VALUES_MELCEPLOGF0_TEMPLATE = """ |
| <td>{max_log_f0}</td> |
| <td>{max_mel_cep_distortion}</td> |
| <td>{max_single_error}</td> |
| """ |
| |
| MELCEPLOGF0_BASELINE_TEMPLATE = """{val:.2E}""" |
| MELCEPLOGF0_DIFF_TEMPLATE = \ |
| """{val:.2E} <span class='{span}'>({diff:.1f}%)</span>""" |
| |
| |
| ACCURACY_HEADERS_PHONE_ERROR_RATE_TEMPLATE = """ |
| <th>Max phone error rate</th> |
| <th>Max scalar error</th> |
| """ |
| |
| ACCURACY_VALUES_PHONE_ERROR_RATE_TEMPLATE = """ |
| <td>{max_phone_error_rate}</td> |
| <td>{max_single_error}</td> |
| """ |
| |
| PHONE_ERROR_RATE_BASELINE_TEMPLATE = """{val:.3f}""" |
| PHONE_ERROR_RATE_DIFF_TEMPLATE = \ |
| """{val:.3f} <span class='{span}'>({diff:.1f}%)</span>""" |
| |
| |
| ACCURACY_HEADERS_BASIC_TEMPLATE = """ |
| <th>Max single scalar error</th> |
| """ |
| |
| |
| ACCURACY_VALUES_BASIC_TEMPLATE = """ |
| <td>{max_single_error:.2f}</td> |
| """ |
| |
| CHART_JS_FILE = 'Chart.bundle.min.js' |
| |
| if __name__ == '__main__': |
| main() |
