| #!/usr/bin/env python3 |
| # -*- coding: utf-8 -*- |
| #*************************************************************************** |
| # _ _ ____ _ |
| # Project ___| | | | _ \| | |
| # / __| | | | |_) | | |
| # | (__| |_| | _ <| |___ |
| # \___|\___/|_| \_\_____| |
| # |
| # Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. |
| # |
| # This software is licensed as described in the file COPYING, which |
| # you should have received as part of this distribution. The terms |
| # are also available at https://curl.se/docs/copyright.html. |
| # |
| # You may opt to use, copy, modify, merge, publish, distribute and/or sell |
| # copies of the Software, and permit persons to whom the Software is |
| # furnished to do so, under the terms of the COPYING file. |
| # |
| # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY |
| # KIND, either express or implied. |
| # |
| # SPDX-License-Identifier: curl |
| # |
| ########################################################################### |
| # |
| import argparse |
| import json |
| import logging |
| import os |
| import re |
| import sys |
| from statistics import mean |
| from typing import Dict, Any, Optional, List |
| |
| from testenv import Env, Httpd, Nghttpx, CurlClient, Caddy, ExecResult, NghttpxQuic, RunProfile |
| |
| log = logging.getLogger(__name__) |
| |
| |
| class ScoreCardException(Exception): |
| pass |
| |
| |
| class ScoreCard: |
| |
| def __init__(self, env: Env, |
| httpd: Optional[Httpd], |
| nghttpx: Optional[Nghttpx], |
| caddy: Optional[Caddy], |
| verbose: int, |
| curl_verbose: int): |
| self.verbose = verbose |
| self.env = env |
| self.httpd = httpd |
| self.nghttpx = nghttpx |
| self.caddy = caddy |
| self._silent_curl = not curl_verbose |
| |
| def info(self, msg): |
| if self.verbose > 0: |
| sys.stderr.write(msg) |
| sys.stderr.flush() |
| |
| def handshakes(self, proto: str) -> Dict[str, Any]: |
| props = {} |
| sample_size = 5 |
| self.info(f'TLS Handshake\n') |
| for authority in [ |
| 'curl.se', 'google.com', 'cloudflare.com', 'nghttp2.org' |
| ]: |
| self.info(f' {authority}...') |
| props[authority] = {} |
| for ipv in ['ipv4', 'ipv6']: |
| self.info(f'{ipv}...') |
| c_samples = [] |
| hs_samples = [] |
| errors = [] |
| for i in range(sample_size): |
| curl = CurlClient(env=self.env, silent=self._silent_curl) |
| args = [ |
| '--http3-only' if proto == 'h3' else '--http2', |
| f'--{ipv}', f'https://{authority}/' |
| ] |
| r = curl.run_direct(args=args, with_stats=True) |
| if r.exit_code == 0 and len(r.stats) == 1: |
| c_samples.append(r.stats[0]['time_connect']) |
| hs_samples.append(r.stats[0]['time_appconnect']) |
| else: |
| errors.append(f'exit={r.exit_code}') |
| props[authority][f'{ipv}-connect'] = mean(c_samples) \ |
| if len(c_samples) else -1 |
| props[authority][f'{ipv}-handshake'] = mean(hs_samples) \ |
| if len(hs_samples) else -1 |
| props[authority][f'{ipv}-errors'] = errors |
| self.info('ok.\n') |
| return props |
| |
| def _make_docs_file(self, docs_dir: str, fname: str, fsize: int): |
| fpath = os.path.join(docs_dir, fname) |
| data1k = 1024*'x' |
| flen = 0 |
| with open(fpath, 'w') as fd: |
| while flen < fsize: |
| fd.write(data1k) |
| flen += len(data1k) |
| return flen |
| |
| def _check_downloads(self, r: ExecResult, count: int): |
| error = '' |
| if r.exit_code != 0: |
| error += f'exit={r.exit_code} ' |
| if r.exit_code != 0 or len(r.stats) != count: |
| error += f'stats={len(r.stats)}/{count} ' |
| fails = [s for s in r.stats if s['response_code'] != 200] |
| if len(fails) > 0: |
| error += f'{len(fails)} failed' |
| return error if len(error) > 0 else None |
| |
| def transfer_single(self, url: str, proto: str, count: int): |
| sample_size = count |
| count = 1 |
| samples = [] |
| errors = [] |
| profiles = [] |
| self.info(f'single...') |
| for i in range(sample_size): |
| curl = CurlClient(env=self.env, silent=self._silent_curl) |
| r = curl.http_download(urls=[url], alpn_proto=proto, no_save=True, |
| with_headers=False, with_profile=True) |
| err = self._check_downloads(r, count) |
| if err: |
| errors.append(err) |
| else: |
| total_size = sum([s['size_download'] for s in r.stats]) |
| samples.append(total_size / r.duration.total_seconds()) |
| profiles.append(r.profile) |
| return { |
| 'count': count, |
| 'samples': sample_size, |
| 'speed': mean(samples) if len(samples) else -1, |
| 'errors': errors, |
| 'stats': RunProfile.AverageStats(profiles), |
| } |
| |
| def transfer_serial(self, url: str, proto: str, count: int): |
| sample_size = 1 |
| samples = [] |
| errors = [] |
| profiles = [] |
| url = f'{url}?[0-{count - 1}]' |
| self.info(f'serial...') |
| for i in range(sample_size): |
| curl = CurlClient(env=self.env, silent=self._silent_curl) |
| r = curl.http_download(urls=[url], alpn_proto=proto, no_save=True, |
| with_headers=False, with_profile=True) |
| err = self._check_downloads(r, count) |
| if err: |
| errors.append(err) |
| else: |
| total_size = sum([s['size_download'] for s in r.stats]) |
| samples.append(total_size / r.duration.total_seconds()) |
| profiles.append(r.profile) |
| return { |
| 'count': count, |
| 'samples': sample_size, |
| 'speed': mean(samples) if len(samples) else -1, |
| 'errors': errors, |
| 'stats': RunProfile.AverageStats(profiles), |
| } |
| |
| def transfer_parallel(self, url: str, proto: str, count: int): |
| sample_size = 1 |
| samples = [] |
| errors = [] |
| profiles = [] |
| url = f'{url}?[0-{count - 1}]' |
| self.info(f'parallel...') |
| for i in range(sample_size): |
| curl = CurlClient(env=self.env, silent=self._silent_curl) |
| r = curl.http_download(urls=[url], alpn_proto=proto, no_save=True, |
| with_headers=False, |
| with_profile=True, |
| extra_args=['--parallel', |
| '--parallel-max', str(count)]) |
| err = self._check_downloads(r, count) |
| if err: |
| errors.append(err) |
| else: |
| total_size = sum([s['size_download'] for s in r.stats]) |
| samples.append(total_size / r.duration.total_seconds()) |
| profiles.append(r.profile) |
| return { |
| 'count': count, |
| 'samples': sample_size, |
| 'speed': mean(samples) if len(samples) else -1, |
| 'errors': errors, |
| 'stats': RunProfile.AverageStats(profiles), |
| } |
| |
| def download_url(self, label: str, url: str, proto: str, count: int): |
| self.info(f' {count}x{label}: ') |
| props = { |
| 'single': self.transfer_single(url=url, proto=proto, count=10), |
| } |
| if count > 1: |
| props['serial'] = self.transfer_serial(url=url, proto=proto, |
| count=count) |
| props['parallel'] = self.transfer_parallel(url=url, proto=proto, |
| count=count) |
| self.info(f'ok.\n') |
| return props |
| |
| def downloads(self, proto: str, count: int, |
| fsizes: List[int]) -> Dict[str, Any]: |
| scores = {} |
| if self.httpd: |
| if proto == 'h3': |
| port = self.env.h3_port |
| via = 'nghttpx' |
| descr = f'port {port}, proxying httpd' |
| else: |
| port = self.env.https_port |
| via = 'httpd' |
| descr = f'port {port}' |
| self.info(f'{via} downloads\n') |
| scores[via] = { |
| 'description': descr, |
| } |
| for fsize in fsizes: |
| label = self.fmt_size(fsize) |
| fname = f'score{label}.data' |
| self._make_docs_file(docs_dir=self.httpd.docs_dir, |
| fname=fname, fsize=fsize) |
| url = f'https://{self.env.domain1}:{port}/{fname}' |
| results = self.download_url(label=label, url=url, |
| proto=proto, count=count) |
| scores[via][label] = results |
| if self.caddy: |
| port = self.caddy.port |
| via = 'caddy' |
| descr = f'port {port}' |
| self.info('caddy downloads\n') |
| scores[via] = { |
| 'description': descr, |
| } |
| for fsize in fsizes: |
| label = self.fmt_size(fsize) |
| fname = f'score{label}.data' |
| self._make_docs_file(docs_dir=self.caddy.docs_dir, |
| fname=fname, fsize=fsize) |
| url = f'https://{self.env.domain1}:{port}/{fname}' |
| results = self.download_url(label=label, url=url, |
| proto=proto, count=count) |
| scores[via][label] = results |
| return scores |
| |
| def do_requests(self, url: str, proto: str, count: int, |
| max_parallel: int = 1): |
| sample_size = 1 |
| samples = [] |
| errors = [] |
| profiles = [] |
| url = f'{url}?[0-{count - 1}]' |
| extra_args = ['--parallel', '--parallel-max', str(max_parallel)] \ |
| if max_parallel > 1 else [] |
| self.info(f'{max_parallel}...') |
| for i in range(sample_size): |
| curl = CurlClient(env=self.env, silent=self._silent_curl) |
| r = curl.http_download(urls=[url], alpn_proto=proto, no_save=True, |
| with_headers=False, with_profile=True, |
| extra_args=extra_args) |
| err = self._check_downloads(r, count) |
| if err: |
| errors.append(err) |
| else: |
| for _ in r.stats: |
| samples.append(count / r.duration.total_seconds()) |
| profiles.append(r.profile) |
| return { |
| 'count': count, |
| 'samples': sample_size, |
| 'speed': mean(samples) if len(samples) else -1, |
| 'errors': errors, |
| 'stats': RunProfile.AverageStats(profiles), |
| } |
| |
| def requests_url(self, url: str, proto: str, count: int): |
| self.info(f' {url}: ') |
| props = { |
| '1': self.do_requests(url=url, proto=proto, count=count), |
| '6': self.do_requests(url=url, proto=proto, count=count, |
| max_parallel=6), |
| '25': self.do_requests(url=url, proto=proto, count=count, |
| max_parallel=25), |
| '50': self.do_requests(url=url, proto=proto, count=count, |
| max_parallel=50), |
| '100': self.do_requests(url=url, proto=proto, count=count, |
| max_parallel=100), |
| } |
| self.info(f'ok.\n') |
| return props |
| |
| def requests(self, proto: str, req_count) -> Dict[str, Any]: |
| scores = {} |
| if self.httpd: |
| if proto == 'h3': |
| port = self.env.h3_port |
| via = 'nghttpx' |
| descr = f'port {port}, proxying httpd' |
| else: |
| port = self.env.https_port |
| via = 'httpd' |
| descr = f'port {port}' |
| self.info(f'{via} requests\n') |
| self._make_docs_file(docs_dir=self.httpd.docs_dir, |
| fname='reqs10.data', fsize=10*1024) |
| url1 = f'https://{self.env.domain1}:{port}/reqs10.data' |
| scores[via] = { |
| 'description': descr, |
| 'count': req_count, |
| '10KB': self.requests_url(url=url1, proto=proto, count=req_count), |
| } |
| if self.caddy: |
| port = self.caddy.port |
| via = 'caddy' |
| descr = f'port {port}' |
| self.info('caddy requests\n') |
| self._make_docs_file(docs_dir=self.caddy.docs_dir, |
| fname='req10.data', fsize=10 * 1024) |
| url1 = f'https://{self.env.domain1}:{port}/req10.data' |
| scores[via] = { |
| 'description': descr, |
| 'count': req_count, |
| '10KB': self.requests_url(url=url1, proto=proto, count=req_count), |
| } |
| return scores |
| |
| def score_proto(self, proto: str, |
| handshakes: bool = True, |
| downloads: Optional[List[int]] = None, |
| download_count: int = 50, |
| req_count=5000, |
| requests: bool = True): |
| self.info(f"scoring {proto}\n") |
| p = {} |
| if proto == 'h3': |
| p['name'] = 'h3' |
| if not self.env.have_h3_curl(): |
| raise ScoreCardException('curl does not support HTTP/3') |
| for lib in ['ngtcp2', 'quiche', 'msh3', 'nghttp3']: |
| if self.env.curl_uses_lib(lib): |
| p['implementation'] = lib |
| break |
| elif proto == 'h2': |
| p['name'] = 'h2' |
| if not self.env.have_h2_curl(): |
| raise ScoreCardException('curl does not support HTTP/2') |
| for lib in ['nghttp2', 'hyper']: |
| if self.env.curl_uses_lib(lib): |
| p['implementation'] = lib |
| break |
| elif proto == 'h1' or proto == 'http/1.1': |
| proto = 'http/1.1' |
| p['name'] = proto |
| p['implementation'] = 'hyper' if self.env.curl_uses_lib('hyper')\ |
| else 'native' |
| else: |
| raise ScoreCardException(f"unknown protocol: {proto}") |
| |
| if 'implementation' not in p: |
| raise ScoreCardException(f'did not recognized {p} lib') |
| p['version'] = Env.curl_lib_version(p['implementation']) |
| |
| score = { |
| 'curl': self.env.curl_fullname(), |
| 'os': self.env.curl_os(), |
| 'protocol': p, |
| } |
| if handshakes: |
| score['handshakes'] = self.handshakes(proto=proto) |
| if downloads and len(downloads) > 0: |
| score['downloads'] = self.downloads(proto=proto, |
| count=download_count, |
| fsizes=downloads) |
| if requests: |
| score['requests'] = self.requests(proto=proto, req_count=req_count) |
| self.info("\n") |
| return score |
| |
| def fmt_ms(self, tval): |
| return f'{int(tval*1000)} ms' if tval >= 0 else '--' |
| |
| def fmt_size(self, val): |
| if val >= (1024*1024*1024): |
| return f'{val / (1024*1024*1024):0.000f}GB' |
| elif val >= (1024 * 1024): |
| return f'{val / (1024*1024):0.000f}MB' |
| elif val >= 1024: |
| return f'{val / 1024:0.000f}KB' |
| else: |
| return f'{val:0.000f}B' |
| |
| def fmt_mbs(self, val): |
| return f'{val/(1024*1024):0.000f} MB/s' if val >= 0 else '--' |
| |
| def fmt_reqs(self, val): |
| return f'{val:0.000f} r/s' if val >= 0 else '--' |
| |
| def print_score(self, score): |
| print(f'{score["protocol"]["name"].upper()} in {score["curl"]}') |
| if 'handshakes' in score: |
| print(f'{"Handshakes":<24} {"ipv4":25} {"ipv6":28}') |
| print(f' {"Host":<17} {"Connect":>12} {"Handshake":>12} ' |
| f'{"Connect":>12} {"Handshake":>12} {"Errors":<20}') |
| for key, val in score["handshakes"].items(): |
| print(f' {key:<17} {self.fmt_ms(val["ipv4-connect"]):>12} ' |
| f'{self.fmt_ms(val["ipv4-handshake"]):>12} ' |
| f'{self.fmt_ms(val["ipv6-connect"]):>12} ' |
| f'{self.fmt_ms(val["ipv6-handshake"]):>12} ' |
| f'{"/".join(val["ipv4-errors"] + val["ipv6-errors"]):<20}' |
| ) |
| if 'downloads' in score: |
| # get the key names of all sizes and measurements made |
| sizes = [] |
| measures = [] |
| m_names = {} |
| mcol_width = 12 |
| mcol_sw = 17 |
| for server, server_score in score['downloads'].items(): |
| for sskey, ssval in server_score.items(): |
| if isinstance(ssval, str): |
| continue |
| if sskey not in sizes: |
| sizes.append(sskey) |
| for mkey, mval in server_score[sskey].items(): |
| if mkey not in measures: |
| measures.append(mkey) |
| m_names[mkey] = f'{mkey}({mval["count"]}x)' |
| |
| print('Downloads') |
| print(f' {"Server":<8} {"Size":>8}', end='') |
| for m in measures: print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='') |
| print(f' {"Errors":^20}') |
| |
| for server in score['downloads']: |
| for size in sizes: |
| size_score = score['downloads'][server][size] |
| print(f' {server:<8} {size:>8}', end='') |
| errors = [] |
| for key, val in size_score.items(): |
| if 'errors' in val: |
| errors.extend(val['errors']) |
| for m in measures: |
| if m in size_score: |
| print(f' {self.fmt_mbs(size_score[m]["speed"]):>{mcol_width}}', end='') |
| s = f'[{size_score[m]["stats"]["cpu"]:>.1f}%'\ |
| f'/{self.fmt_size(size_score[m]["stats"]["rss"])}]' |
| print(f' {s:<{mcol_sw}}', end='') |
| else: |
| print(' '*mcol_width, end='') |
| if len(errors): |
| print(f' {"/".join(errors):<20}') |
| else: |
| print(f' {"-":^20}') |
| |
| if 'requests' in score: |
| sizes = [] |
| measures = [] |
| m_names = {} |
| mcol_width = 9 |
| mcol_sw = 13 |
| for server in score['requests']: |
| server_score = score['requests'][server] |
| for sskey, ssval in server_score.items(): |
| if isinstance(ssval, str) or isinstance(ssval, int): |
| continue |
| if sskey not in sizes: |
| sizes.append(sskey) |
| for mkey, mval in server_score[sskey].items(): |
| if mkey not in measures: |
| measures.append(mkey) |
| m_names[mkey] = f'{mkey}' |
| |
| print('Requests, max in parallel') |
| print(f' {"Server":<8} {"Size":>6} {"Reqs":>6}', end='') |
| for m in measures: print(f' {m_names[m]:>{mcol_width}} {"[cpu/rss]":<{mcol_sw}}', end='') |
| print(f' {"Errors":^10}') |
| |
| for server in score['requests']: |
| for size in sizes: |
| size_score = score['requests'][server][size] |
| count = score['requests'][server]['count'] |
| print(f' {server:<8} {size:>6} {count:>6}', end='') |
| errors = [] |
| for key, val in size_score.items(): |
| if 'errors' in val: |
| errors.extend(val['errors']) |
| for m in measures: |
| if m in size_score: |
| print(f' {self.fmt_reqs(size_score[m]["speed"]):>{mcol_width}}', end='') |
| s = f'[{size_score[m]["stats"]["cpu"]:>.1f}%'\ |
| f'/{self.fmt_size(size_score[m]["stats"]["rss"])}]' |
| print(f' {s:<{mcol_sw}}', end='') |
| else: |
| print(' '*mcol_width, end='') |
| if len(errors): |
| print(f' {"/".join(errors):<10}') |
| else: |
| print(f' {"-":^10}') |
| |
| |
| def parse_size(s): |
| m = re.match(r'(\d+)(mb|kb|gb)?', s, re.IGNORECASE) |
| if m is None: |
| raise Exception(f'unrecognized size: {s}') |
| size = int(m.group(1)) |
| if not m.group(2): |
| pass |
| elif m.group(2).lower() == 'kb': |
| size *= 1024 |
| elif m.group(2).lower() == 'mb': |
| size *= 1024 * 1024 |
| elif m.group(2).lower() == 'gb': |
| size *= 1024 * 1024 * 1024 |
| return size |
| |
| |
| def main(): |
| parser = argparse.ArgumentParser(prog='scorecard', description=""" |
| Run a range of tests to give a scorecard for a HTTP protocol |
| 'h3' or 'h2' implementation in curl. |
| """) |
| parser.add_argument("-v", "--verbose", action='count', default=1, |
| help="log more output on stderr") |
| parser.add_argument("-j", "--json", action='store_true', |
| default=False, help="print json instead of text") |
| parser.add_argument("-H", "--handshakes", action='store_true', |
| default=False, help="evaluate handshakes only") |
| parser.add_argument("-d", "--downloads", action='store_true', |
| default=False, help="evaluate downloads") |
| parser.add_argument("--download", action='append', type=str, |
| default=None, help="evaluate download size") |
| parser.add_argument("--download-count", action='store', type=int, |
| default=50, help="perform that many downloads") |
| parser.add_argument("-r", "--requests", action='store_true', |
| default=False, help="evaluate requests") |
| parser.add_argument("--request-count", action='store', type=int, |
| default=5000, help="perform that many requests") |
| parser.add_argument("--httpd", action='store_true', default=False, |
| help="evaluate httpd server only") |
| parser.add_argument("--caddy", action='store_true', default=False, |
| help="evaluate caddy server only") |
| parser.add_argument("--curl-verbose", action='store_true', |
| default=False, help="run curl with `-v`") |
| parser.add_argument("protocol", default='h2', nargs='?', |
| help="Name of protocol to score") |
| args = parser.parse_args() |
| |
| if args.verbose > 0: |
| console = logging.StreamHandler() |
| console.setLevel(logging.INFO) |
| console.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) |
| logging.getLogger('').addHandler(console) |
| |
| protocol = args.protocol |
| handshakes = True |
| downloads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024] |
| if args.download is not None: |
| downloads = [] |
| for x in args.download: |
| downloads.extend([parse_size(s) for s in x.split(',')]) |
| requests = True |
| if args.downloads or args.requests or args.handshakes: |
| handshakes = args.handshakes |
| if not args.downloads: |
| downloads = None |
| requests = args.requests |
| |
| test_httpd = protocol != 'h3' |
| test_caddy = True |
| if args.caddy or args.httpd: |
| test_caddy = args.caddy |
| test_httpd = args.httpd |
| |
| rv = 0 |
| env = Env() |
| env.setup() |
| env.test_timeout = None |
| httpd = None |
| nghttpx = None |
| caddy = None |
| try: |
| if test_httpd: |
| httpd = Httpd(env=env) |
| assert httpd.exists(), \ |
| f'httpd not found: {env.httpd}' |
| httpd.clear_logs() |
| assert httpd.start() |
| if 'h3' == protocol: |
| nghttpx = NghttpxQuic(env=env) |
| nghttpx.clear_logs() |
| assert nghttpx.start() |
| if test_caddy and env.caddy: |
| caddy = Caddy(env=env) |
| caddy.clear_logs() |
| assert caddy.start() |
| |
| card = ScoreCard(env=env, httpd=httpd, nghttpx=nghttpx, caddy=caddy, |
| verbose=args.verbose, curl_verbose=args.curl_verbose) |
| score = card.score_proto(proto=protocol, |
| handshakes=handshakes, |
| downloads=downloads, |
| download_count=args.download_count, |
| req_count=args.request_count, |
| requests=requests) |
| if args.json: |
| print(json.JSONEncoder(indent=2).encode(score)) |
| else: |
| card.print_score(score) |
| |
| except ScoreCardException as ex: |
| sys.stderr.write(f"ERROR: {str(ex)}\n") |
| rv = 1 |
| except KeyboardInterrupt: |
| log.warning("aborted") |
| rv = 1 |
| finally: |
| if caddy: |
| caddy.stop() |
| if nghttpx: |
| nghttpx.stop(wait_dead=False) |
| if httpd: |
| httpd.stop() |
| sys.exit(rv) |
| |
| |
| if __name__ == "__main__": |
| main() |
