apps/CameraITS/tools/run_all_tests.py - platform/cts - Git at Google

 # Copyright 2014 The Android Open Source Project
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 import os
 import os.path
 import tempfile
 import subprocess
 import time
 import sys
 import textwrap
 import its.device
 from its.device import ItsSession

 def main():
     """Run all the automated tests, saving intermediate files, and producing
     a summary/report of the results.

     Script should be run from the top-level CameraITS directory.

     Command line Arguments:
         camera: the camera(s) to be tested. Use comma to separate multiple
                 camera Ids. Ex: "camera=0,1" or "camera=1"
         scenes: the test scene(s) to be executed. Use comma to separate multiple
                 scenes. Ex: "scenes=scene0,scene1" or "scenes=0,1,sensor_fusion"
                 (sceneX can be abbreviated by X where X is a integer)
     """

     SKIP_RET_CODE = 101

     # Not yet mandated tests
     NOT_YET_MANDATED = {
         "scene0":[
             "test_jitter"
         ],
         "scene1":[
             "test_ae_precapture_trigger",
             "test_crop_region_raw",
             "test_ev_compensation_advanced",
             "test_ev_compensation_basic",
             "test_yuv_plus_jpeg"
         ],
         "scene2":[],
         "scene3":[],
         "scene4":[],
         "scene5":[],
         "sensor_fusion":[]
     }

     all_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4", "scene5",
               "sensor_fusion"]

     scene_req = {
         "scene0" : None,
         "scene1" : "A grey card covering at least the middle 30% of the scene",
         "scene2" : "A picture containing human faces",
         "scene3" : "A chart containing sharp edges like ISO 12233",
         "scene4" : "A specific test page of a circle covering at least the "
                    "middle 50% of the scene. See CameraITS.pdf section 2.3.4 "
                    "for more details",
         "scene5" : "Capture images with a diffuser attached to the camera. See "
                    "CameraITS.pdf section 2.3.4 for more details",
         "sensor_fusion" : "Rotating checkboard pattern. See "
                           "sensor_fusion/SensorFusion.pdf for detailed "
                           "instructions. Note that this test will be skipped "
                           "on devices not supporting REALTIME camera timestamp."
                           "If that is the case, no scene setup is required and "
                           "you can just answer Y when being asked if the scene "
                           "is okay"
     }
     scene_extra_args = {
         "scene5" : ["doAF=False"]
     }

     camera_ids = []
     scenes = []
     for s in sys.argv[1:]:
         if s[:7] == "camera=" and len(s) > 7:
             camera_ids = s[7:].split(',')
         elif s[:7] == "scenes=" and len(s) > 7:
             scenes = s[7:].split(',')

     # Run through all scenes if user does not supply one
     if not scenes:
         scenes = all_scenes
     else:
         # Validate user input scene names
         valid_scenes = True
         temp_scenes = []
         for s in scenes:
             if s in all_scenes:
                 temp_scenes.append(s)
             else:
                 try:
                     # Try replace "X" to "sceneX"
                     scene_num = int(s)
                     scene_str = "scene" + s
                     if scene_str not in all_scenes:
                         valid_scenes = False
                         break
                     temp_scenes.append(scene_str)
                 except ValueError:
                     valid_scenes = False
                     break

         if not valid_scenes:
             print "Unknown scene specifiied:", s
             assert(False)
         scenes = temp_scenes

     # Initialize test results
     results = {}
     result_key = ItsSession.RESULT_KEY
     for s in all_scenes:
         results[s] = {result_key: ItsSession.RESULT_NOT_EXECUTED}

     # Make output directories to hold the generated files.
     topdir = tempfile.mkdtemp()
     print "Saving output files to:", topdir, "\n"

     device_id = its.device.get_device_id()
     device_id_arg = "device=" + device_id
     print "Testing device " + device_id

     # user doesn't specify camera id, run through all cameras
     if not camera_ids:
         camera_ids_path = os.path.join(topdir, "camera_ids.txt")
         out_arg = "out=" + camera_ids_path
         cmd = ['python',
                os.path.join(os.getcwd(),"tools/get_camera_ids.py"), out_arg,
                device_id_arg]
         retcode = subprocess.call(cmd,cwd=topdir)
         assert(retcode == 0)
         with open(camera_ids_path, "r") as f:
             for line in f:
                 camera_ids.append(line.replace('\n', ''))

     print "Running ITS on camera: %s, scene %s" % (camera_ids, scenes)

     for camera_id in camera_ids:
         # Loop capturing images until user confirm test scene is correct
         camera_id_arg = "camera=" + camera_id
         print "Preparing to run ITS on camera", camera_id

         os.mkdir(os.path.join(topdir, camera_id))
         for d in scenes:
             os.mkdir(os.path.join(topdir, camera_id, d))

         for scene in scenes:
             tests = [(s[:-3],os.path.join("tests", scene, s))
                      for s in os.listdir(os.path.join("tests",scene))
                      if s[-3:] == ".py" and s[:4] == "test"]
             tests.sort()

             summary = "Cam" + camera_id + " " + scene + "\n"
             numpass = 0
             numskip = 0
             num_not_mandated_fail = 0
             numfail = 0
             if scene_req[scene] != None:
                 out_path = os.path.join(topdir, camera_id, scene+".jpg")
                 out_arg = "out=" + out_path
                 scene_arg = "scene=" + scene_req[scene]
                 extra_args = scene_extra_args.get(scene, [])
                 cmd = ['python',
                         os.path.join(os.getcwd(),"tools/validate_scene.py"),
                         camera_id_arg, out_arg, scene_arg, device_id_arg] + \
                         extra_args
                 retcode = subprocess.call(cmd,cwd=topdir)
                 assert(retcode == 0)
             print "Start running ITS on camera %s, %s" % (camera_id, scene)

             # Run each test, capturing stdout and stderr.
             for (testname,testpath) in tests:
                 cmd = ['python', os.path.join(os.getcwd(),testpath)] + \
                       sys.argv[1:] + [camera_id_arg]
                 outdir = os.path.join(topdir,camera_id,scene)
                 outpath = os.path.join(outdir,testname+"_stdout.txt")
                 errpath = os.path.join(outdir,testname+"_stderr.txt")
                 t0 = time.time()
                 with open(outpath,"w") as fout, open(errpath,"w") as ferr:
                     retcode = subprocess.call(
                             cmd,stderr=ferr,stdout=fout,cwd=outdir)
                 t1 = time.time()

                 test_failed = False
                 if retcode == 0:
                     retstr = "PASS "
                     numpass += 1
                 elif retcode == SKIP_RET_CODE:
                     retstr = "SKIP "
                     numskip += 1
                 elif retcode != 0 and testname in NOT_YET_MANDATED[scene]:
                     retstr = "FAIL*"
                     num_not_mandated_fail += 1
                 else:
                     retstr = "FAIL "
                     numfail += 1
                     test_failed = True

                 msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0)
                 print msg
                 msg_short = "%s %s [%.1fs]" % (retstr, testname, t1-t0)
                 if test_failed:
                     summary += msg_short + "\n"

             if numskip > 0:
                 skipstr = ", %d test%s skipped" % (
                         numskip, "s" if numskip > 1 else "")
             else:
                 skipstr = ""

             test_result = "\n%d / %d tests passed (%.1f%%)%s" % (
                     numpass + num_not_mandated_fail, len(tests) - numskip,
                     100.0 * float(numpass + num_not_mandated_fail) /
                             (len(tests) - numskip)
                             if len(tests) != numskip else 100.0,
                     skipstr)
             print test_result

             if num_not_mandated_fail > 0:
                 msg = "(*) tests are not yet mandated"
                 print msg

             summary_path = os.path.join(topdir, camera_id, scene, "summary.txt")
             with open(summary_path, "w") as f:
                 f.write(summary)

             passed = numfail == 0
             results[scene][result_key] = (ItsSession.RESULT_PASS if passed
                     else ItsSession.RESULT_FAIL)
             results[scene][ItsSession.SUMMARY_KEY] = summary_path

         print "Reporting ITS result to CtsVerifier"
         its.device.report_result(device_id, camera_id, results)

     print "ITS tests finished. Please go back to CtsVerifier and proceed"

 if __name__ == '__main__':
     main()
	# Copyright 2014 The Android Open Source Project
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	import os
	import os.path
	import tempfile
	import subprocess
	import time
	import sys
	import textwrap
	import its.device
	from its.device import ItsSession

	def main():
	"""Run all the automated tests, saving intermediate files, and producing
	a summary/report of the results.

	Script should be run from the top-level CameraITS directory.

	Command line Arguments:
	camera: the camera(s) to be tested. Use comma to separate multiple
	camera Ids. Ex: "camera=0,1" or "camera=1"
	scenes: the test scene(s) to be executed. Use comma to separate multiple
	scenes. Ex: "scenes=scene0,scene1" or "scenes=0,1,sensor_fusion"
	(sceneX can be abbreviated by X where X is a integer)
	"""

	SKIP_RET_CODE = 101

	# Not yet mandated tests
	NOT_YET_MANDATED = {
	"scene0":[
	"test_jitter"
	],
	"scene1":[
	"test_ae_precapture_trigger",
	"test_crop_region_raw",
	"test_ev_compensation_advanced",
	"test_ev_compensation_basic",
	"test_yuv_plus_jpeg"
	],
	"scene2":[],
	"scene3":[],
	"scene4":[],
	"scene5":[],
	"sensor_fusion":[]
	}

	all_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4", "scene5",
	"sensor_fusion"]

	scene_req = {
	"scene0" : None,
	"scene1" : "A grey card covering at least the middle 30% of the scene",
	"scene2" : "A picture containing human faces",
	"scene3" : "A chart containing sharp edges like ISO 12233",
	"scene4" : "A specific test page of a circle covering at least the "
	"middle 50% of the scene. See CameraITS.pdf section 2.3.4 "
	"for more details",
	"scene5" : "Capture images with a diffuser attached to the camera. See "
	"CameraITS.pdf section 2.3.4 for more details",
	"sensor_fusion" : "Rotating checkboard pattern. See "
	"sensor_fusion/SensorFusion.pdf for detailed "
	"instructions. Note that this test will be skipped "
	"on devices not supporting REALTIME camera timestamp."
	"If that is the case, no scene setup is required and "
	"you can just answer Y when being asked if the scene "
	"is okay"
	}
	scene_extra_args = {
	"scene5" : ["doAF=False"]
	}

	camera_ids = []
	scenes = []
	for s in sys.argv[1:]:
	if s[:7] == "camera=" and len(s) > 7:
	camera_ids = s[7:].split(',')
	elif s[:7] == "scenes=" and len(s) > 7:
	scenes = s[7:].split(',')

	# Run through all scenes if user does not supply one
	if not scenes:
	scenes = all_scenes
	else:
	# Validate user input scene names
	valid_scenes = True
	temp_scenes = []
	for s in scenes:
	if s in all_scenes:
	temp_scenes.append(s)
	else:
	try:
	# Try replace "X" to "sceneX"
	scene_num = int(s)
	scene_str = "scene" + s
	if scene_str not in all_scenes:
	valid_scenes = False
	break
	temp_scenes.append(scene_str)
	except ValueError:
	valid_scenes = False
	break

	if not valid_scenes:
	print "Unknown scene specifiied:", s
	assert(False)
	scenes = temp_scenes

	# Initialize test results
	results = {}
	result_key = ItsSession.RESULT_KEY
	for s in all_scenes:
	results[s] = {result_key: ItsSession.RESULT_NOT_EXECUTED}

	# Make output directories to hold the generated files.
	topdir = tempfile.mkdtemp()
	print "Saving output files to:", topdir, "\n"

	device_id = its.device.get_device_id()
	device_id_arg = "device=" + device_id
	print "Testing device " + device_id

	# user doesn't specify camera id, run through all cameras
	if not camera_ids:
	camera_ids_path = os.path.join(topdir, "camera_ids.txt")
	out_arg = "out=" + camera_ids_path
	cmd = ['python',
	os.path.join(os.getcwd(),"tools/get_camera_ids.py"), out_arg,
	device_id_arg]
	retcode = subprocess.call(cmd,cwd=topdir)
	assert(retcode == 0)
	with open(camera_ids_path, "r") as f:
	for line in f:
	camera_ids.append(line.replace('\n', ''))

	print "Running ITS on camera: %s, scene %s" % (camera_ids, scenes)

	for camera_id in camera_ids:
	# Loop capturing images until user confirm test scene is correct
	camera_id_arg = "camera=" + camera_id
	print "Preparing to run ITS on camera", camera_id

	os.mkdir(os.path.join(topdir, camera_id))
	for d in scenes:
	os.mkdir(os.path.join(topdir, camera_id, d))

	for scene in scenes:
	tests = [(s[:-3],os.path.join("tests", scene, s))
	for s in os.listdir(os.path.join("tests",scene))
	if s[-3:] == ".py" and s[:4] == "test"]
	tests.sort()

	summary = "Cam" + camera_id + " " + scene + "\n"
	numpass = 0
	numskip = 0
	num_not_mandated_fail = 0
	numfail = 0
	if scene_req[scene] != None:
	out_path = os.path.join(topdir, camera_id, scene+".jpg")
	out_arg = "out=" + out_path
	scene_arg = "scene=" + scene_req[scene]
	extra_args = scene_extra_args.get(scene, [])
	cmd = ['python',
	os.path.join(os.getcwd(),"tools/validate_scene.py"),
	camera_id_arg, out_arg, scene_arg, device_id_arg] + \
	extra_args
	retcode = subprocess.call(cmd,cwd=topdir)
	assert(retcode == 0)
	print "Start running ITS on camera %s, %s" % (camera_id, scene)

	# Run each test, capturing stdout and stderr.
	for (testname,testpath) in tests:
	cmd = ['python', os.path.join(os.getcwd(),testpath)] + \
	sys.argv[1:] + [camera_id_arg]
	outdir = os.path.join(topdir,camera_id,scene)
	outpath = os.path.join(outdir,testname+"_stdout.txt")
	errpath = os.path.join(outdir,testname+"_stderr.txt")
	t0 = time.time()
	with open(outpath,"w") as fout, open(errpath,"w") as ferr:
	retcode = subprocess.call(
	cmd,stderr=ferr,stdout=fout,cwd=outdir)
	t1 = time.time()

	test_failed = False
	if retcode == 0:
	retstr = "PASS "
	numpass += 1
	elif retcode == SKIP_RET_CODE:
	retstr = "SKIP "
	numskip += 1
	elif retcode != 0 and testname in NOT_YET_MANDATED[scene]:
	retstr = "FAIL*"
	num_not_mandated_fail += 1
	else:
	retstr = "FAIL "
	numfail += 1
	test_failed = True

	msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0)
	print msg
	msg_short = "%s %s [%.1fs]" % (retstr, testname, t1-t0)
	if test_failed:
	summary += msg_short + "\n"

	if numskip > 0:
	skipstr = ", %d test%s skipped" % (
	numskip, "s" if numskip > 1 else "")
	else:
	skipstr = ""

	test_result = "\n%d / %d tests passed (%.1f%%)%s" % (
	numpass + num_not_mandated_fail, len(tests) - numskip,
	100.0 * float(numpass + num_not_mandated_fail) /
	(len(tests) - numskip)
	if len(tests) != numskip else 100.0,
	skipstr)
	print test_result

	if num_not_mandated_fail > 0:
	msg = "(*) tests are not yet mandated"
	print msg

	summary_path = os.path.join(topdir, camera_id, scene, "summary.txt")
	with open(summary_path, "w") as f:
	f.write(summary)

	passed = numfail == 0
	results[scene][result_key] = (ItsSession.RESULT_PASS if passed
	else ItsSession.RESULT_FAIL)
	results[scene][ItsSession.SUMMARY_KEY] = summary_path

	print "Reporting ITS result to CtsVerifier"
	its.device.report_result(device_id, camera_id, results)

	print "ITS tests finished. Please go back to CtsVerifier and proceed"

	if __name__ == '__main__':
	main()