CameraITS: various ITS improvement
1. run_all_tests.py now don't require a camera id argument.
The script now iterate through all cameras automatically.
2. run_all_tests.py will now repeat capturing a image for
user to visually check if current device placement is good
for ITS testing. This should help front camera test since
front camera will be facing test scene.
3. CtsVerifier will now save ITS test summaries and reported
errors.
4. Enable pass button if all cameras on device are legacy level.
Change-Id: Ic8740f65dbaa67b0dc2d2aa8737355c0dffe1b6f
diff --git a/apps/CameraITS/CameraITS.pdf b/apps/CameraITS/CameraITS.pdf
index 0d10bae..2430420 100644
--- a/apps/CameraITS/CameraITS.pdf
+++ b/apps/CameraITS/CameraITS.pdf
Binary files differ
diff --git a/apps/CameraITS/pymodules/its/device.py b/apps/CameraITS/pymodules/its/device.py
index beba0ae..035e70b 100644
--- a/apps/CameraITS/pymodules/its/device.py
+++ b/apps/CameraITS/pymodules/its/device.py
@@ -53,7 +53,9 @@
PACKAGE = 'com.android.cts.verifier.camera.its'
INTENT_START = 'com.android.cts.verifier.camera.its.START'
ACTION_ITS_RESULT = 'com.android.cts.verifier.camera.its.ACTION_ITS_RESULT'
+ EXTRA_CAMERA_ID = 'camera.its.extra.CAMERA_ID'
EXTRA_SUCCESS = 'camera.its.extra.SUCCESS'
+ EXTRA_SUMMARY = 'camera.its.extra.SUMMARY'
# TODO: Handle multiple connected devices.
ADB = "adb -d"
@@ -241,6 +243,20 @@
raise its.error.Error('Invalid command response')
return data['objValue']
+ def get_camera_ids(self):
+ """Get a list of camera device Ids that can be opened.
+
+ Returns:
+ a list of camera ID string
+ """
+ cmd = {}
+ cmd["cmdName"] = "getCameraIds"
+ self.sock.send(json.dumps(cmd) + "\n")
+ data,_ = self.__read_response_from_socket()
+ if data['tag'] != 'cameraIds':
+ raise its.error.Error('Invalid command response')
+ return data['objValue']['cameraIdArray']
+
def get_camera_properties(self):
"""Get the camera properties object for the device.
@@ -510,21 +526,32 @@
rets.append(objs if ncap>1 else objs[0])
return rets if len(rets)>1 else rets[0]
-def report_result(camera_id, success):
+def report_result(camera_id, success, summary_path=None):
"""Send a pass/fail result to the device, via an intent.
Args:
camera_id: The ID string of the camera for which to report pass/fail.
success: Boolean, indicating if the result was pass or fail.
+ summary_path: (Optional) path to ITS summary file on host PC
Returns:
Nothing.
"""
- resultstr = "%s=%s" % (camera_id, 'True' if success else 'False')
- _run(('%s shell am broadcast '
- '-a %s --es %s %s') % (ItsSession.ADB, ItsSession.ACTION_ITS_RESULT,
- ItsSession.EXTRA_SUCCESS, resultstr))
-
+ device_summary_path = "/sdcard/camera_" + camera_id + "_its_summary.txt"
+ if summary_path is not None:
+ _run("%s push %s %s" % (
+ ItsSession.ADB, summary_path, device_summary_path))
+ _run("%s shell am broadcast -a %s --es %s %s --es %s %s --es %s %s" % (
+ ItsSession.ADB, ItsSession.ACTION_ITS_RESULT,
+ ItsSession.EXTRA_CAMERA_ID, camera_id,
+ ItsSession.EXTRA_SUCCESS, 'True' if success else 'False',
+ ItsSession.EXTRA_SUMMARY, device_summary_path))
+ else:
+ _run("%s shell am broadcast -a %s --es %s %s --es %s %s --es %s %s" % (
+ ItsSession.ADB, ItsSession.ACTION_ITS_RESULT,
+ ItsSession.EXTRA_CAMERA_ID, camera_id,
+ ItsSession.EXTRA_SUCCESS, 'True' if success else 'False',
+ ItsSession.EXTRA_SUMMARY, "null"))
def _run(cmd):
"""Replacement for os.system, with hiding of stdout+stderr messages.
diff --git a/apps/CameraITS/tools/get_camera_ids.py b/apps/CameraITS/tools/get_camera_ids.py
new file mode 100644
index 0000000..010b046
--- /dev/null
+++ b/apps/CameraITS/tools/get_camera_ids.py
@@ -0,0 +1,37 @@
+# Copyright 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import its.device
+import its.objects
+import its.image
+
+def main():
+ """get camera ids and save it to disk.
+ """
+ out_path = ""
+ for s in sys.argv[1:]:
+ if s[:4] == "out=" and len(s) > 4:
+ out_path = s[4:]
+ # kind of weird we need to open a camera to get camera ids, but
+ # this is how ITS is working now.
+ with its.device.ItsSession() as cam:
+ camera_ids = cam.get_camera_ids()
+ if out_path != "":
+ with open(out_path, "w") as f:
+ for camera_id in camera_ids:
+ f.write(camera_id + "\n")
+
+if __name__ == '__main__':
+ main()
diff --git a/apps/CameraITS/tools/run_all_tests.py b/apps/CameraITS/tools/run_all_tests.py
index f5a53b1..b56281d 100644
--- a/apps/CameraITS/tools/run_all_tests.py
+++ b/apps/CameraITS/tools/run_all_tests.py
@@ -18,6 +18,7 @@
import subprocess
import time
import sys
+import textwrap
import its.device
def main():
@@ -57,62 +58,111 @@
# Make output directories to hold the generated files.
topdir = tempfile.mkdtemp()
- for d in scenes:
- os.mkdir(os.path.join(topdir, d))
print "Saving output files to:", topdir, "\n"
- # determine camera id
- camera_id = 0
+ camera_ids = []
for s in sys.argv[1:]:
if s[:7] == "camera=" and len(s) > 7:
- camera_id = s[7:]
+ camera_ids.append(s[7:])
- # Run each test, capturing stdout and stderr.
- numpass = 0
- numskip = 0
- numnotmandatedfail = 0
- numfail = 0
- for (scene,testname,testpath) in tests:
- cmd = ['python', os.path.join(os.getcwd(),testpath)] + sys.argv[1:]
- outdir = os.path.join(topdir,scene)
- outpath = os.path.join(outdir,testname+"_stdout.txt")
- errpath = os.path.join(outdir,testname+"_stderr.txt")
- t0 = time.time()
- with open(outpath,"w") as fout, open(errpath,"w") as ferr:
- retcode = subprocess.call(cmd,stderr=ferr,stdout=fout,cwd=outdir)
- t1 = time.time()
+ # user doesn't specify camera id, run through all cameras
+ if not camera_ids:
+ camera_ids_path = os.path.join(topdir, "camera_ids.txt")
+ out_arg = "out=" + camera_ids_path
+ cmd = ['python',
+ os.path.join(os.getcwd(),"tools/get_camera_ids.py"), out_arg]
+ retcode = subprocess.call(cmd,cwd=topdir)
+ assert(retcode == 0)
+ with open(camera_ids_path, "r") as f:
+ for line in f:
+ camera_ids.append(line.replace('\n', ''))
- if retcode == 0:
- retstr = "PASS "
- numpass += 1
- elif retcode == SKIP_RET_CODE:
- retstr = "SKIP "
- numskip += 1
- elif retcode != 0 and testname in NOT_YET_MANDATED[scene]:
- retstr = "FAIL*"
- numnotmandatedfail += 1
+ print "Running ITS on the following cameras:", camera_ids
+
+ for camera_id in camera_ids:
+ # Loop capturing images until user confirm test scene is correct
+ camera_id_arg = "camera=" + camera_id
+ print "Preparing to run ITS on camera", camera_id
+
+ os.mkdir(os.path.join(topdir, camera_id))
+ for d in scenes:
+ os.mkdir(os.path.join(topdir, camera_id, d))
+
+ out_path = os.path.join(topdir, camera_id, "scene.jpg")
+ out_arg = "out=" + out_path
+ cmd = ['python',
+ os.path.join(os.getcwd(),"tools/validate_scene.py"),
+ camera_id_arg, out_arg]
+ retcode = subprocess.call(cmd,cwd=topdir)
+ assert(retcode == 0)
+
+ print "Start running ITS on camera: ", camera_id
+ # Run each test, capturing stdout and stderr.
+ summary = "ITS test result summary for camera " + camera_id + "\n"
+ numpass = 0
+ numskip = 0
+ numnotmandatedfail = 0
+ numfail = 0
+
+ for (scene,testname,testpath) in tests:
+ cmd = ['python', os.path.join(os.getcwd(),testpath)] + \
+ sys.argv[1:] + [camera_id_arg]
+ outdir = os.path.join(topdir,camera_id,scene)
+ outpath = os.path.join(outdir,testname+"_stdout.txt")
+ errpath = os.path.join(outdir,testname+"_stderr.txt")
+ t0 = time.time()
+ with open(outpath,"w") as fout, open(errpath,"w") as ferr:
+ retcode = subprocess.call(cmd,stderr=ferr,stdout=fout,cwd=outdir)
+ t1 = time.time()
+
+ if retcode == 0:
+ retstr = "PASS "
+ numpass += 1
+ elif retcode == SKIP_RET_CODE:
+ retstr = "SKIP "
+ numskip += 1
+ elif retcode != 0 and testname in NOT_YET_MANDATED[scene]:
+ retstr = "FAIL*"
+ numnotmandatedfail += 1
+ else:
+ retstr = "FAIL "
+ numfail += 1
+
+ msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0)
+ print msg
+ summary += msg + "\n"
+ if retcode != 0 and retcode != SKIP_RET_CODE:
+ # Dump the stderr if the test fails
+ with open (errpath, "r") as error_file:
+ errors = error_file.read()
+ summary += errors + "\n"
+
+ if numskip > 0:
+ skipstr = ", %d test%s skipped" % (numskip, "s" if numskip > 1 else "")
else:
- retstr = "FAIL "
- numfail += 1
+ skipstr = ""
- print "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0)
+ test_result = "\n%d / %d tests passed (%.1f%%)%s" % (
+ numpass + numnotmandatedfail, len(tests) - numskip,
+ 100.0 * float(numpass + numnotmandatedfail) / (len(tests) - numskip)
+ if len(tests) != numskip else 100.0,
+ skipstr)
+ print test_result
+ summary += test_result + "\n"
- if numskip > 0:
- skipstr = ", %d test%s skipped" % (numskip, "s" if numskip > 1 else "")
- else:
- skipstr = ""
+ if numnotmandatedfail > 0:
+ msg = "(*) tests are not yet mandated"
+ print msg
+ summary += msg + "\n"
- print "\n%d / %d tests passed (%.1f%%)%s" % (
- numpass + numnotmandatedfail, len(tests) - numskip,
- 100.0 * float(numpass + numnotmandatedfail) / (len(tests) - numskip)
- if len(tests) != numskip else 100.0,
- skipstr)
+ result = numfail == 0
+ print "Reporting ITS result to CtsVerifier"
+ summary_path = os.path.join(topdir, camera_id, "summary.txt")
+ with open(summary_path, "w") as f:
+ f.write(summary)
+ its.device.report_result(camera_id, result, summary_path)
- if numnotmandatedfail > 0:
- print "(*) tests are not yet mandated"
-
- its.device.report_result(camera_id, numfail == 0)
+ print "ITS tests finished. Please go back to CtsVerifier and proceed"
if __name__ == '__main__':
main()
-
diff --git a/apps/CameraITS/tools/validate_scene.py b/apps/CameraITS/tools/validate_scene.py
new file mode 100644
index 0000000..e1e89f2
--- /dev/null
+++ b/apps/CameraITS/tools/validate_scene.py
@@ -0,0 +1,60 @@
+# Copyright 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import its.device
+import its.objects
+import its.image
+
+def main():
+ """capture a yuv image and save it to argv[1]
+ """
+ camera_id = -1
+ out_path = ""
+ for s in sys.argv[1:]:
+ if s[:7] == "camera=" and len(s) > 7:
+ camera_id = s[7:]
+ elif s[:4] == "out=" and len(s) > 4:
+ out_path = s[4:]
+
+ if camera_id == -1:
+ print "Error: need to specify which camera to use"
+ assert(False)
+
+ with its.device.ItsSession() as cam:
+ raw_input("Press Enter after placing camera " + camera_id +
+ " to frame the test scene")
+ # Converge 3A prior to capture.
+ cam.do_3a(do_af=True, lock_ae=True, lock_awb=True)
+ props = cam.get_camera_properties()
+ req = its.objects.fastest_auto_capture_request(props)
+ req["android.control.awbLock"] = True
+ req["android.control.aeLock"] = True
+ while True:
+ print "Capture an image to check the test scene"
+ cap = cam.do_capture(req)
+ img = its.image.convert_capture_to_rgb_image(cap)
+ if out_path != "":
+ its.image.write_image(img, out_path)
+ print "Please check scene setup in", out_path
+ choice = raw_input(
+ "Is the image okay for ITS scene1? (Y/N)").lower()
+ if choice == "y":
+ break
+ else:
+ raw_input("Press Enter after placing camera " + camera_id +
+ " to frame the test scene")
+
+if __name__ == '__main__':
+ main()
diff --git a/apps/CtsVerifier/res/values/strings.xml b/apps/CtsVerifier/res/values/strings.xml
index bf07e8f..a19bcec 100644
--- a/apps/CtsVerifier/res/values/strings.xml
+++ b/apps/CtsVerifier/res/values/strings.xml
@@ -787,7 +787,7 @@
\n\n3. Setup the test scene described in the CameraITS README file, and aim the camera
at it.
\n\n4. Run the full ITS test suite on all possible camera Ids.
- (cd CameraITS; python tools/run_all_tests.py camera=[cameraId]). Once all
+ (cd CameraITS; python tools/run_all_tests.py). Once all
of the tests have been run, the \'PASS\' button will be enabled if all of the tests have
succeeded. Please note that these tests can take 20+ minutes to run.
</string>
diff --git a/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsService.java b/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsService.java
index a305cd2..0fda75b 100644
--- a/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsService.java
+++ b/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsService.java
@@ -544,6 +544,8 @@
doCapture(cmdObj);
} else if ("doVibrate".equals(cmdObj.getString("cmdName"))) {
doVibrate(cmdObj);
+ } else if ("getCameraIds".equals(cmdObj.getString("cmdName"))) {
+ doGetCameraIds();
} else {
throw new ItsException("Unknown command: " + cmd);
}
@@ -729,6 +731,38 @@
mSocketRunnableObj.sendResponse(mCameraCharacteristics);
}
+ private void doGetCameraIds() throws ItsException {
+ String[] devices;
+ try {
+ devices = mCameraManager.getCameraIdList();
+ if (devices == null || devices.length == 0) {
+ throw new ItsException("No camera devices");
+ }
+ } catch (CameraAccessException e) {
+ throw new ItsException("Failed to get device ID list", e);
+ }
+
+ try {
+ JSONObject obj = new JSONObject();
+ JSONArray array = new JSONArray();
+ for (String id : devices) {
+ CameraCharacteristics characteristics = mCameraManager.getCameraCharacteristics(id);
+ // Only supply camera Id for non-legacy cameras since legacy camera does not
+ // support ITS
+ if (characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL) !=
+ CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY) {
+ array.put(id);
+ }
+ }
+ obj.put("cameraIdArray", array);
+ mSocketRunnableObj.sendResponse("cameraIds", obj);
+ } catch (org.json.JSONException e) {
+ throw new ItsException("JSON error: ", e);
+ } catch (android.hardware.camera2.CameraAccessException e) {
+ throw new ItsException("Access error: ", e);
+ }
+ }
+
private void prepareCaptureReader(int[] widths, int[] heights, int formats[], int numSurfaces) {
if (mCaptureReaders != null) {
for (int i = 0; i < mCaptureReaders.length; i++) {
diff --git a/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsTestActivity.java b/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsTestActivity.java
index 12b9bfc..17df106 100644
--- a/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsTestActivity.java
+++ b/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsTestActivity.java
@@ -27,9 +27,19 @@
import android.os.Bundle;
import android.util.Log;
import android.widget.Toast;
-import java.util.HashSet;
-import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.HashMap;
+import java.util.Arrays;
+import java.util.List;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import com.android.compatibility.common.util.ResultType;
+import com.android.compatibility.common.util.ResultUnit;
import com.android.cts.verifier.PassFailButtons;
import com.android.cts.verifier.R;
@@ -41,7 +51,9 @@
*/
public class ItsTestActivity extends PassFailButtons.Activity {
private static final String TAG = "ItsTestActivity";
+ private static final String EXTRA_CAMERA_ID = "camera.its.extra.CAMERA_ID";
private static final String EXTRA_SUCCESS = "camera.its.extra.SUCCESS";
+ private static final String EXTRA_SUMMARY = "camera.its.extra.SUMMARY";
private static final String ACTION_ITS_RESULT =
"com.android.cts.verifier.camera.its.ACTION_ITS_RESULT";
@@ -50,20 +62,32 @@
public void onReceive(Context context, Intent intent) {
Log.i(TAG, "Received result for Camera ITS tests");
if (ACTION_ITS_RESULT.equals(intent.getAction())) {
+ String cameraId = intent.getStringExtra(EXTRA_CAMERA_ID);
String result = intent.getStringExtra(EXTRA_SUCCESS);
- String[] parts = result.split("=");
- if (parts.length != 2) {
- Toast.makeText(ItsTestActivity.this,
- "Received unknown ITS result string: " + result,
- Toast.LENGTH_SHORT).show();
+ String summaryPath = intent.getStringExtra(EXTRA_SUMMARY);
+ if (!mNonLegacyCameraIds.contains(cameraId)) {
+ Log.e(TAG, "Unknown camera id " + cameraId + " reported to ITS");
+ return;
}
- String cameraId = parts[0];
- boolean pass = parts[1].equals("True");
+
+ Log.i(TAG, "ITS summary path is: " + summaryPath);
+ mSummaryMap.put(cameraId, summaryPath);
+ // Create summary report
+ if (mSummaryMap.keySet().containsAll(mNonLegacyCameraIds)) {
+ StringBuilder summary = new StringBuilder();
+ for (String id : mNonLegacyCameraIds) {
+ String path = mSummaryMap.get(id);
+ appendFileContentToSummary(summary, path);
+ }
+ ItsTestActivity.this.getReportLog().setSummary(
+ summary.toString(), 1.0, ResultType.NEUTRAL, ResultUnit.NONE);
+ }
+ boolean pass = result.equals("True");
if(pass) {
Log.i(TAG, "Received Camera " + cameraId + " ITS SUCCESS from host.");
mITSPassedCameraIds.add(cameraId);
- if (mCameraIds != null &&
- mITSPassedCameraIds.containsAll(Arrays.asList(mCameraIds))) {
+ if (mNonLegacyCameraIds != null && mNonLegacyCameraIds.size() != 0 &&
+ mITSPassedCameraIds.containsAll(mNonLegacyCameraIds)) {
ItsTestActivity.this.showToast(R.string.its_test_passed);
ItsTestActivity.this.getPassButton().setEnabled(true);
}
@@ -73,11 +97,40 @@
}
}
}
+
+ private void appendFileContentToSummary(StringBuilder summary, String path) {
+ BufferedReader reader = null;
+ try {
+ reader = new BufferedReader(new FileReader(path));
+ String line = null;
+ do {
+ line = reader.readLine();
+ if (line != null) {
+ summary.append(line);
+ }
+ } while (line != null);
+ } catch (FileNotFoundException e) {
+ Log.e(TAG, "Cannot find ITS summary file at " + path);
+ summary.append("Cannot find ITS summary file at " + path);
+ } catch (IOException e) {
+ Log.e(TAG, "IO exception when trying to read " + path);
+ summary.append("IO exception when trying to read " + path);
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ }
+ }
+ }
+ }
}
private final SuccessReceiver mSuccessReceiver = new SuccessReceiver();
private final HashSet<String> mITSPassedCameraIds = new HashSet<>();
- private String[] mCameraIds = null;
+ // map camera id to ITS summary report path
+ private final HashMap<String, String> mSummaryMap = new HashMap<>();
+ ArrayList<String> mNonLegacyCameraIds = null;
@Override
protected void onCreate(Bundle savedInstanceState) {
@@ -96,25 +149,27 @@
showToast(R.string.no_camera_manager);
} else {
try {
- mCameraIds = manager.getCameraIdList();
+ String[] cameraIds = manager.getCameraIdList();
+ mNonLegacyCameraIds = new ArrayList<String>();
boolean allCamerasAreLegacy = true;
- for (String id : mCameraIds) {
+ for (String id : cameraIds) {
CameraCharacteristics characteristics = manager.getCameraCharacteristics(id);
if (characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)
!= CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY) {
+ mNonLegacyCameraIds.add(id);
allCamerasAreLegacy = false;
- break;
}
}
if (allCamerasAreLegacy) {
showToast(R.string.all_legacy_devices);
- getPassButton().setEnabled(false);
+ getPassButton().setEnabled(true);
}
} catch (CameraAccessException e) {
Toast.makeText(ItsTestActivity.this,
"Received error from camera service while checking device capabilities: "
+ e, Toast.LENGTH_SHORT).show();
}
+ Log.d(TAG, "register ITS result receiver");
IntentFilter filter = new IntentFilter(ACTION_ITS_RESULT);
registerReceiver(mSuccessReceiver, filter);
}
@@ -123,6 +178,7 @@
@Override
protected void onPause() {
super.onPause();
+ Log.d(TAG, "unregister ITS result receiver");
unregisterReceiver(mSuccessReceiver);
}
