blob: c065f1246779720c7751d76de6e428f6888c72e9 [file] [log] [blame]
# Copyright 2014 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import os.path
import tempfile
import subprocess
import time
import sys
import textwrap
import its.device
def main():
"""Run all the automated tests, saving intermediate files, and producing
a summary/report of the results.
Script should be run from the top-level CameraITS directory.
"""
SKIP_RET_CODE = 101
# Not yet mandated tests
NOT_YET_MANDATED = {
"scene0":[
"test_jitter"
],
"scene1":[
"test_ae_precapture_trigger",
"test_crop_region_raw",
"test_ev_compensation_advanced",
"test_ev_compensation_basic",
"test_yuv_plus_jpeg"
],
"scene2":[]
}
# Get all the scene0 and scene1 tests, which can be run using the same
# physical setup.
scenes = ["scene0", "scene1", "scene2"]
scene_req = {
"scene0" : None,
"scene1" : "A grey card covering at least the middle 30% of the scene",
"scene2" : "A picture containing human faces"
}
tests = []
for d in scenes:
tests += [(d,s[:-3],os.path.join("tests", d, s))
for s in os.listdir(os.path.join("tests",d))
if s[-3:] == ".py"]
tests.sort()
# Make output directories to hold the generated files.
topdir = tempfile.mkdtemp()
print "Saving output files to:", topdir, "\n"
camera_ids = []
for s in sys.argv[1:]:
if s[:7] == "camera=" and len(s) > 7:
camera_ids.append(s[7:])
# user doesn't specify camera id, run through all cameras
if not camera_ids:
camera_ids_path = os.path.join(topdir, "camera_ids.txt")
out_arg = "out=" + camera_ids_path
cmd = ['python',
os.path.join(os.getcwd(),"tools/get_camera_ids.py"), out_arg]
retcode = subprocess.call(cmd,cwd=topdir)
assert(retcode == 0)
with open(camera_ids_path, "r") as f:
for line in f:
camera_ids.append(line.replace('\n', ''))
print "Running ITS on the following cameras:", camera_ids
for camera_id in camera_ids:
# Loop capturing images until user confirm test scene is correct
camera_id_arg = "camera=" + camera_id
print "Preparing to run ITS on camera", camera_id
os.mkdir(os.path.join(topdir, camera_id))
for d in scenes:
os.mkdir(os.path.join(topdir, camera_id, d))
print "Start running ITS on camera: ", camera_id
# Run each test, capturing stdout and stderr.
summary = "ITS test result summary for camera " + camera_id + "\n"
numpass = 0
numskip = 0
numnotmandatedfail = 0
numfail = 0
prev_scene = ""
for (scene,testname,testpath) in tests:
if scene != prev_scene and scene_req[scene] != None:
out_path = os.path.join(topdir, camera_id, scene+".jpg")
out_arg = "out=" + out_path
scene_arg = "scene=" + scene_req[scene]
cmd = ['python',
os.path.join(os.getcwd(),"tools/validate_scene.py"),
camera_id_arg, out_arg, scene_arg]
retcode = subprocess.call(cmd,cwd=topdir)
assert(retcode == 0)
print "Start running tests for", scene
prev_scene = scene
cmd = ['python', os.path.join(os.getcwd(),testpath)] + \
sys.argv[1:] + [camera_id_arg]
outdir = os.path.join(topdir,camera_id,scene)
outpath = os.path.join(outdir,testname+"_stdout.txt")
errpath = os.path.join(outdir,testname+"_stderr.txt")
t0 = time.time()
with open(outpath,"w") as fout, open(errpath,"w") as ferr:
retcode = subprocess.call(cmd,stderr=ferr,stdout=fout,cwd=outdir)
t1 = time.time()
if retcode == 0:
retstr = "PASS "
numpass += 1
elif retcode == SKIP_RET_CODE:
retstr = "SKIP "
numskip += 1
elif retcode != 0 and testname in NOT_YET_MANDATED[scene]:
retstr = "FAIL*"
numnotmandatedfail += 1
else:
retstr = "FAIL "
numfail += 1
msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0)
print msg
summary += msg + "\n"
if retcode != 0 and retcode != SKIP_RET_CODE:
# Dump the stderr if the test fails
with open (errpath, "r") as error_file:
errors = error_file.read()
summary += errors + "\n"
if numskip > 0:
skipstr = ", %d test%s skipped" % (numskip, "s" if numskip > 1 else "")
else:
skipstr = ""
test_result = "\n%d / %d tests passed (%.1f%%)%s" % (
numpass + numnotmandatedfail, len(tests) - numskip,
100.0 * float(numpass + numnotmandatedfail) / (len(tests) - numskip)
if len(tests) != numskip else 100.0,
skipstr)
print test_result
summary += test_result + "\n"
if numnotmandatedfail > 0:
msg = "(*) tests are not yet mandated"
print msg
summary += msg + "\n"
result = numfail == 0
print "Reporting ITS result to CtsVerifier"
summary_path = os.path.join(topdir, camera_id, "summary.txt")
with open(summary_path, "w") as f:
f.write(summary)
its.device.report_result(camera_id, result, summary_path)
print "ITS tests finished. Please go back to CtsVerifier and proceed"
if __name__ == '__main__':
main()