blob: b890ea6a43bd1b5a930fa2fc4f0dcf467241da2a [file] [log] [blame]
#!/usr/bin/env python3
import argparse
import bz2
import json
import subprocess
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple
import boto3 # type: ignore[import]
import botocore # type: ignore[import]
def get_git_commit_history(
*,
path: str,
ref: str
) -> List[Tuple[str, datetime]]:
rc = subprocess.check_output(
['git', '-C', path, 'log', '--pretty=format:%H %ct', ref],
).decode("latin-1")
return [
(x[0], datetime.fromtimestamp(int(x[1])))
for x in [line.split(" ") for line in rc.split("\n")]
]
def get_ossci_jsons(
*,
bucket: Any,
sha: str,
jobs: Optional[List[str]]
) -> Dict[str, Any]:
prefix = f"test_time/{sha}/"
objs: List[Any]
if jobs is None:
objs = list(bucket.objects.filter(Prefix=prefix))
else:
objs = []
for job in jobs:
objs.extend(list(bucket.objects.filter(Prefix=f"{prefix}{job}/")))
# initial pass to avoid downloading more than necessary
# in the case where there are multiple reports for a single sha+job
uniqueified = {obj.key.split('/')[2]: obj for obj in objs}
return {
job: json.loads(bz2.decompress(obj.get()['Body'].read()))
for job, obj in uniqueified.items()
}
def get_case(
*,
data: Any,
suite_name: str,
test_name: str,
) -> Optional[Dict[str, Any]]:
suite = data.get('suites', {}).get(suite_name)
if suite:
testcase_times = {
case['name']: case
for case in suite['cases']
}
return testcase_times.get(test_name)
return None
def case_status(case: Dict[str, Any]) -> Optional[str]:
for k in {'errored', 'failed', 'skipped'}:
if case[k]:
return k
return None
def make_column(
*,
data: Any,
suite_name: str,
test_name: str,
digits: int,
) -> str:
decimals = 3
num_length = digits + 1 + decimals
case = get_case(data=data, suite_name=suite_name, test_name=test_name)
if case:
status = case_status(case)
if status:
return f'{status.rjust(num_length)} '
else:
return f'{case["seconds"]:{num_length}.{decimals}f}s'
return ' ' * (num_length + 1)
def make_columns(
*,
bucket: Any,
sha: str,
jobs: List[str],
suite_name: str,
test_name: str,
digits: int,
) -> str:
jsons = get_ossci_jsons(bucket=bucket, sha=sha, jobs=jobs)
return ' '.join(
make_column(
data=jsons.get(job, {}),
suite_name=suite_name,
test_name=test_name,
digits=digits,
)
for job in jobs
)
def make_lines(
*,
bucket: Any,
sha: str,
jobs: Optional[List[str]],
suite_name: str,
test_name: str,
) -> List[str]:
jsons = get_ossci_jsons(bucket=bucket, sha=sha, jobs=jobs)
lines = []
for job, data in jsons.items():
case = get_case(data=data, suite_name=suite_name, test_name=test_name)
if case:
status = case_status(case)
lines.append(f'{job} {case["seconds"]} {status or ""}')
return lines
def display_history(
*,
bucket: Any,
commits: List[Tuple[str, datetime]],
jobs: Optional[List[str]],
suite_name: str,
test_name: str,
delta: int,
mode: str,
digits: int,
) -> None:
prev_time = datetime.now()
for sha, time in commits:
if (prev_time - time).total_seconds() < delta * 3600:
continue
prev_time = time
lines: List[str]
if mode == 'columns':
assert jobs is not None
lines = [make_columns(
bucket=bucket,
sha=sha,
jobs=jobs,
suite_name=suite_name,
test_name=test_name,
digits=digits,
)]
else:
assert mode == 'multiline'
lines = make_lines(
bucket=bucket,
sha=sha,
jobs=jobs,
suite_name=suite_name,
test_name=test_name,
)
for line in lines:
print(f"{time} {sha} {line}".rstrip())
if __name__ == "__main__":
parser = argparse.ArgumentParser(
__file__,
description='Display the history of a test.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
'mode',
choices=['columns', 'multiline'],
help='output format',
)
parser.add_argument(
'--pytorch',
help='path to local PyTorch clone',
default='.',
)
parser.add_argument(
'--ref',
help='starting point (most recent Git ref) to display history for',
default='master',
)
parser.add_argument(
'--delta',
type=int,
help='minimum number of hours between rows',
default=12,
)
parser.add_argument(
'--digits',
type=int,
help='(columns) number of digits to display before the decimal point',
default=4,
)
parser.add_argument(
'--all',
action='store_true',
help='(multiline) ignore listed jobs, show all jobs for each commit',
)
parser.add_argument(
'suite',
help='name of the suite containing the test',
)
parser.add_argument(
'test',
help='name of the test',
)
parser.add_argument(
'job',
nargs='*',
help='names of jobs to display columns for, in order',
default=[],
)
args = parser.parse_args()
commits = get_git_commit_history(path=args.pytorch, ref=args.ref)
s3 = boto3.resource("s3", config=botocore.config.Config(signature_version=botocore.UNSIGNED))
bucket = s3.Bucket('ossci-metrics')
display_history(
bucket=bucket,
commits=commits,
jobs=None if args.all else args.job,
suite_name=args.suite,
test_name=args.test,
delta=args.delta,
mode=args.mode,
digits=args.digits,
)