| #!/usr/bin/env python3 |
| # Copyright (C) 2015 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| import argparse, json, math, re, sys, zipfile |
| import xml.etree.ElementTree as ET |
| from collections import defaultdict, namedtuple |
| |
| |
| class Size(namedtuple('Size', ['width', 'height'])): |
| """A namedtuple with width and height fields.""" |
| def __str__(self): |
| return '%dx%d' % (self.width, self.height) |
| |
| def nicekey(v): |
| """Returns a nicer sort key for sorting strings. |
| |
| This sorts using lower case, with numbers in numerical order first.""" |
| key = [] |
| num = False |
| for p in re.split('(\d+)', v.lower()): |
| if num: |
| key.append(('0', int(p))) |
| elif p: |
| key.append((p, 0)) |
| num = not num |
| return key + [(v, 0)] |
| |
| def nice(v): |
| """Returns a nicer representation for objects in debug messages. |
| |
| Dictionaries are sorted, size is WxH, unicode removed, and floats have 1 digit precision.""" |
| if isinstance(v, dict): |
| return 'dict(' + ', '.join(k + '=' + nice(v) for k, v in sorted(v.items(), key=lambda i: nicekey(i[0]))) + ')' |
| if isinstance(v, str): |
| return repr(v) |
| if isinstance(v, int): |
| return str(v) |
| if isinstance(v, Size): |
| return repr(str(v)) |
| if isinstance(v, float): |
| return '%.1f' % v |
| if isinstance(v, type(u'')): |
| return repr(str(v)) |
| raise ValueError(v) |
| |
| class ResultParser: |
| @staticmethod |
| def _intify(value): |
| """Returns a value converted to int if possible, else the original value.""" |
| try: |
| return int(value) |
| except ValueError: |
| return value |
| |
| def _parseDict(self, value): |
| """Parses a MediaFormat from its string representation sans brackets.""" |
| return dict((k, self._intify(v)) |
| for k, v in re.findall(r'([^ =]+)=([^ [=]+(?:|\[[^\]]+\]))(?:, |$)', value)) |
| |
| def _cleanFormat(self, format): |
| """Removes internal fields from a parsed MediaFormat.""" |
| format.pop('what', None) |
| format.pop('image-data', None) |
| |
| MESSAGE_PATTERN = r'(?P<key>\w+)=(?P<value>\{[^}]*\}|[^ ,{}]+)' |
| |
| def _parsePartialResult(self, message_match): |
| """Parses a partial test result conforming to the message pattern. |
| |
| Returns: |
| A tuple of string key and int, string or dict value, where dict has |
| string keys mapping to int or string values. |
| """ |
| key, value = message_match.group('key', 'value') |
| if value.startswith('{'): |
| value = self._parseDict(value[1:-1]) |
| if key.endswith('Format'): |
| self._cleanFormat(value) |
| else: |
| value = self._intify(value) |
| return key, value |
| |
| |
| def perc(data, p, fn=round): |
| """Returns a percentile value from a sorted array. |
| |
| Arguments: |
| data: sorted data |
| p: percentile value (0-100) |
| fn: method used for rounding the percentile to an integer index in data |
| """ |
| return data[int(fn((len(data) - 1) * p / 100))] |
| |
| |
| def genXml(data, A=None): |
| yield '<?xml version="1.0" encoding="utf-8" ?>' |
| yield '<!-- Copyright 2016 The Android Open Source Project' |
| yield '' |
| yield ' Licensed under the Apache License, Version 2.0 (the "License");' |
| yield ' you may not use this file except in compliance with the License.' |
| yield ' You may obtain a copy of the License at' |
| yield '' |
| yield ' http://www.apache.org/licenses/LICENSE-2.0' |
| yield '' |
| yield ' Unless required by applicable law or agreed to in writing, software' |
| yield ' distributed under the License is distributed on an "AS IS" BASIS,' |
| yield ' WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.' |
| yield ' See the License for the specific language governing permissions and' |
| yield ' limitations under the License.' |
| yield '-->' |
| yield '' |
| yield '<MediaCodecs>' |
| last_section = None |
| from collections import namedtuple |
| Comp = namedtuple('Comp', 'is_decoder google mime name') |
| Result = namedtuple('Result', 'mn mx p95 med geo p5') |
| for comp_, cdata in sorted(data.items()): |
| comp = Comp(*comp_) |
| section = 'Decoders' if comp.is_decoder else 'Encoders' |
| if section != last_section: |
| if last_section: |
| yield ' </%s>' % last_section |
| yield ' <%s>' % section |
| last_section = section |
| yield ' <MediaCodec name="%s" type="%s" update="true">' % (comp.name, comp.mime) |
| for size, sdata in sorted(cdata.items()): |
| data = sorted(sdata) |
| N = len(data) |
| mn, mx = data[0], data[-1] |
| |
| if N < 20 and not A.ignore: |
| raise ValueError("need at least 20 data points for %s size %s; have %s" % |
| (comp.name, size, N)) |
| |
| TO = 2.2 # tolerance with margin |
| T = TO / 1.1 # tolerance without margin |
| |
| Final = namedtuple('Final', 'comment c2 var qual') |
| lastFinal = None |
| for RG in (10, 15, 20, 25, 30, 40, 50): |
| P = 50./RG |
| quality = 0 |
| p95, med, p5 = perc(data, P, math.floor), perc(data, 50, round), perc(data, 100 - P, math.ceil) |
| geo = math.sqrt(p5 * p95) |
| comment = '' |
| pub_lo, pub_hi = min(int(p95 * T), round(geo)), max(math.ceil(p5 / T), round(geo)) |
| if pub_lo > med: |
| if pub_lo > med * 1.1: |
| quality += 0.5 |
| comment += ' SLOW' |
| pub_lo = int(med) |
| if N < 2 * RG: |
| comment += ' N=%d' % N |
| quality += 2 |
| RGVAR = False |
| if p5 / p95 > T ** 3: |
| quality += 3 |
| RGVAR = True |
| if pub_hi > pub_lo * TO: |
| quality += 1 |
| if RG == 10: |
| # find best pub_lo and pub_hi |
| for i in range(N // 2): |
| pub_lo_, pub_hi_ = min(int(data[N // 2 - i - 1] * T), round(geo), int(med)), max(math.ceil(data[N // 2 + i] / T), round(geo)) |
| if pub_hi_ > pub_lo_ * TO: |
| # ??? |
| pub_lo = min(pub_lo, math.ceil(pub_hi_ / TO)) |
| break |
| pub_lo, pub_hi = pub_lo_, pub_hi_ |
| if mn < pub_lo / T or mx > pub_hi * T or pub_lo <= pub_hi / T: |
| quality += 1 |
| comment += ' FLAKY(' |
| if round(mn, 1) < pub_lo / T: |
| comment += 'mn=%.1f < ' % mn |
| comment += 'RANGE' |
| if round(mx, 1) > pub_hi * T: |
| comment += ' < mx=%.1f' % mx |
| comment += ')' |
| if False: |
| comment += ' DATA(mn=%1.f p%d=%1.f accept=%1.f-%1.f p50=%1.f p%d=%1.f mx=%1.f)' % ( |
| mn, 100-P, p95, pub_lo / T, pub_hi * T, med, P, p5, mx) |
| var = math.sqrt(p5/p95) |
| if p95 < geo / T or p5 > geo * T: |
| if RGVAR: |
| comment += ' RG.VARIANCE:%.1f' % ((p5/p95) ** (1./3)) |
| else: |
| comment += ' variance:%.1f' % var |
| comment = comment.replace('RANGE', '%d - %d' % (math.ceil(pub_lo / T), int(pub_hi * T))) |
| c2 = '' |
| if N >= 2 * RG: |
| c2 += ' N=%d' % N |
| if var <= T or p5 / p95 > T ** 3: |
| c2 += ' v%d%%=%.1f' % (round(100 - 2 * P), var) |
| if A and A.dbg: |
| c2 += ' E=%s' % (str(quality)) |
| if c2: |
| c2 = ' <!--%s -->' % c2 |
| |
| if comment: |
| comment = ' <!-- measured %d%%:%d-%d med:%d%s -->' % (round(100 - 2 * P), int(p95), math.ceil(p5), int(round(med)), comment) |
| if A and A.dbg: yield '<!-- --> %s%s' % (comment, c2) |
| c2 = ' <Limit name="measured-frame-rate-%s" range="%d-%d" />%s' % (size, pub_lo, pub_hi, c2) |
| final = Final(comment, c2, var, quality) |
| if lastFinal and final.var > lastFinal.var * math.sqrt(1.3): |
| if A and A.dbg: yield '<!-- RANGE JUMP -->' |
| break |
| elif not lastFinal or quality <= lastFinal.qual: |
| lastFinal = final |
| if N < 2 * RG or quality >= 4: |
| break |
| comment, c2, var, quality = lastFinal |
| |
| if comment: |
| yield comment |
| yield c2 |
| yield ' </MediaCodec>' |
| if last_section: |
| yield ' </%s>' % last_section |
| yield '</MediaCodecs>' |
| |
| |
| class Data: |
| def __init__(self): |
| self.data = set() |
| self.kind = {} |
| self.devices = set() |
| self.parser = ResultParser() |
| |
| def summarize(self, A=None): |
| devs = sorted(self.devices) |
| # device > (not encoder,goog,mime,codec) > size > fps |
| xmlInfo = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) |
| |
| for mime, encoder, goog in sorted(set(self.kind.values())): |
| for dev, build, codec, size, num, std, avg, p0, p5, p10, p20, p30, p40, p50, p60, p70, p80, p90, p95, p100 in self.data: |
| if self.kind[codec] != (mime, encoder, goog): |
| continue |
| |
| if p95 > 2: # ignore measurements at or below 2fps |
| xmlInfo[dev][(not encoder, goog, mime, codec)][size].append(p95) |
| else: |
| print("warning: p95 value is suspiciously low: {}".format( |
| nice(dict(config=dict(dev=dev, codec=codec, size=str(size), N=num), |
| data=dict(std=std, avg=avg, p0=p0, p5=p5, p10=p10, p20=p20, p30=p30, p40=p40, |
| p50=p50, p60=p60, p70=p70, p80=p80, p90=p90, p95=p95, p100=p100)))), |
| file=sys.stderr) |
| for dev, ddata in xmlInfo.items(): |
| outFile = '{}.media_codecs_performance.xml'.format(dev) |
| print(f"generating {outFile}", file=sys.stderr) |
| with open(outFile, "wt") as out: |
| for l in genXml(ddata, A=A): |
| out.write(l + '\n') |
| print(l) |
| print(f"generated {outFile}", file=sys.stderr) |
| |
| def parse_fmt(self, fmt): |
| return self.parser._parseDict(fmt) |
| |
| def parse_perf(self, a, device, build): |
| def rateFn(i): |
| if i is None: |
| return i |
| elif i == 0: |
| return 1e6 |
| return 1000. / i |
| |
| points = ('avg', 'min', 'p5', 'p10', 'p20', 'p30', 'p40', 'p50', 'p60', 'p70', 'p80', 'p90', 'p95', 'max') |
| a = dict(a) |
| codec = a['codec_name'] + '' |
| mime = a['mime_type'] |
| size = Size(a['width'], a['height']) |
| if 'decode_to' in a: |
| fmt = self.parse_fmt(a['output_format']) |
| ofmt = self.parse_fmt(a['input_format']) |
| else: |
| fmt = self.parse_fmt(a['input_format']) |
| ofmt = self.parse_fmt(a['output_format']) |
| size = Size(max(fmt['width'], ofmt['width']), max(fmt['height'], ofmt['height'])) |
| |
| try: |
| prefix = 'time_avg_stats_' |
| if prefix + 'stdev' in a and a[prefix + 'avg']: |
| stdev = (a[prefix + 'stdev'] * 1e3 / a[prefix + 'avg'] ** 2) |
| data = ((device, build, codec, size, a[prefix + 'num'], stdev) + |
| tuple(rateFn(a.get(prefix + i)) for i in points)) |
| self.data.add(data) |
| self.kind[codec] = (mime, 'decode_to' not in a, codec.lower().startswith('omx.google.')) |
| self.devices.add(data[0]) |
| except (KeyError, ZeroDivisionError): |
| print(a, file=sys.stderr) |
| raise |
| |
| def parse_json(self, json, device, build): |
| for test, results in json: |
| if test in ("video_encoder_performance", "video_decoder_performance"): |
| try: |
| if isinstance(results, list) and len(results[0]) and len(results[0][0]) == 2 and len(results[0][0][0]): |
| for result in results: |
| self.parse_perf(result, device, build) |
| else: |
| self.parse_perf(results, device, build) |
| except KeyboardInterrupt: |
| raise |
| |
| def parse_result(self, result): |
| device, build = '', '' |
| if not result.endswith('.zip'): |
| print(f"cannot parse %{result}", file=sys.stderr) |
| return |
| |
| try: |
| with zipfile.ZipFile(result) as zip: |
| resultInfo, testInfos = None, [] |
| for info in zip.infolist(): |
| if re.search(r'/GenericDeviceInfo.deviceinfo.json$', info.filename): |
| resultInfo = info |
| elif re.search(r'/Cts(Media|Video)(Decoder)?TestCases\.reportlog\.json$', info.filename): |
| testInfos.append(info) |
| if resultInfo: |
| try: |
| jsonFile = zip.open(resultInfo) |
| jsonData = json.load(jsonFile) |
| device, build = jsonData['build_device'], jsonData['build_id'] |
| except ValueError: |
| print(f"could not parse %{resultInfo.filename}", file=sys.stderr) |
| for info in testInfos: |
| jsonFile = zip.open(info) |
| try: |
| jsonData = json.load(jsonFile, object_pairs_hook=lambda items: items) |
| except ValueError: |
| print(f"cannot parse JSON in {info.filename}", file=sys.stderr) |
| self.parse_json(jsonData, device, build) |
| |
| except zipfile.BadZipfile: |
| raise ValueError('bad zipfile') |
| |
| |
| P = argparse.ArgumentParser("gar_v2") |
| P.add_argument("--dbg", "-v", action='store_true', help="dump debug info into xml") |
| P.add_argument("--ignore", "-I", action='store_true', help="ignore minimum sample count") |
| P.add_argument("result_zip", nargs="*") |
| A = P.parse_args() |
| |
| D = Data() |
| for res in A.result_zip: |
| D.parse_result(res) |
| D.summarize(A=A) |
