DO NOT MERGE ANYWHERE: CameraITS: allow each scene to be ran independently

Bug: 30577461
Bug: 28768014

(cherry-picked from 54225abbacfd37d932ceb79b137d4eee10e5caf1)

Change-Id: I1361fcae38e9c0e16b01fbfc4667fb3648005757
diff --git a/apps/CameraITS/pymodules/its/device.py b/apps/CameraITS/pymodules/its/device.py
index 351b03c..692a62d 100644
--- a/apps/CameraITS/pymodules/its/device.py
+++ b/apps/CameraITS/pymodules/its/device.py
@@ -67,9 +67,17 @@
     PACKAGE = 'com.android.cts.verifier.camera.its'
     INTENT_START = 'com.android.cts.verifier.camera.its.START'
     ACTION_ITS_RESULT = 'com.android.cts.verifier.camera.its.ACTION_ITS_RESULT'
+    EXTRA_VERSION = 'camera.its.extra.VERSION'
+    CURRENT_ITS_VERSION = '1.0' # version number to sync with CtsVerifier
     EXTRA_CAMERA_ID = 'camera.its.extra.CAMERA_ID'
-    EXTRA_SUCCESS = 'camera.its.extra.SUCCESS'
-    EXTRA_SUMMARY = 'camera.its.extra.SUMMARY'
+    EXTRA_RESULTS = 'camera.its.extra.RESULTS'
+
+    RESULT_PASS = 'PASS'
+    RESULT_FAIL = 'FAIL'
+    RESULT_NOT_EXECUTED = 'NOT_EXECUTED'
+    RESULT_VALUES = {RESULT_PASS, RESULT_FAIL, RESULT_NOT_EXECUTED}
+    RESULT_KEY = 'result'
+    SUMMARY_KEY = 'summary'
 
     adb = "adb -d"
     device_id = ""
@@ -781,34 +789,43 @@
 
     return device_id
 
-def report_result(device_id, camera_id, success, summary_path=None):
+def report_result(device_id, camera_id, results):
     """Send a pass/fail result to the device, via an intent.
 
     Args:
         device_id: The ID string of the device to report the results to.
         camera_id: The ID string of the camera for which to report pass/fail.
-        success: Boolean, indicating if the result was pass or fail.
-        summary_path: (Optional) path to ITS summary file on host PC
-
+        results: a dictionary contains all ITS scenes as key and result/summary
+                 of current ITS run. See test_report_result unit test for
+                 an example.
     Returns:
         Nothing.
     """
     adb = "adb -s " + device_id
-    device_summary_path = "/sdcard/camera_" + camera_id + "_its_summary.txt"
-    if summary_path is not None:
-        _run("%s push %s %s" % (
-                adb, summary_path, device_summary_path))
-        _run("%s shell am broadcast -a %s --es %s %s --es %s %s --es %s %s" % (
-                adb, ItsSession.ACTION_ITS_RESULT,
-                ItsSession.EXTRA_CAMERA_ID, camera_id,
-                ItsSession.EXTRA_SUCCESS, 'True' if success else 'False',
-                ItsSession.EXTRA_SUMMARY, device_summary_path))
-    else:
-        _run("%s shell am broadcast -a %s --es %s %s --es %s %s --es %s %s" % (
-                adb, ItsSession.ACTION_ITS_RESULT,
-                ItsSession.EXTRA_CAMERA_ID, camera_id,
-                ItsSession.EXTRA_SUCCESS, 'True' if success else 'False',
-                ItsSession.EXTRA_SUMMARY, "null"))
+    # Validate/process results argument
+    for scene in results:
+        result_key = ItsSession.RESULT_KEY
+        summary_key = ItsSession.SUMMARY_KEY
+        if result_key not in results[scene]:
+            raise its.error.Error('ITS result not found for ' + scene)
+        if results[scene][result_key] not in ItsSession.RESULT_VALUES:
+            raise its.error.Error('Unknown ITS result for %s: %s' % (
+                    scene, results[result_key]))
+        if summary_key in results[scene]:
+            device_summary_path = "/sdcard/its_camera%s_%s.txt" % (
+                    camera_id, scene)
+            _run("%s push %s %s" % (
+                    adb, results[scene][summary_key], device_summary_path))
+            results[scene][summary_key] = device_summary_path
+    json_results = json.dumps(results)
+    cmd = "%s shell am broadcast -a %s --es %s %s --es %s %s --es %s \'%s\'" % (
+            adb, ItsSession.ACTION_ITS_RESULT,
+            ItsSession.EXTRA_VERSION, ItsSession.CURRENT_ITS_VERSION,
+            ItsSession.EXTRA_CAMERA_ID, camera_id,
+            ItsSession.EXTRA_RESULTS, json_results)
+    if len(cmd) > 4095:
+        print "ITS command string might be too long! len:", len(cmd)
+    _run(cmd)
 
 def _run(cmd):
     """Replacement for os.system, with hiding of stdout+stderr messages.
@@ -821,8 +838,20 @@
     """Run a suite of unit tests on this module.
     """
 
-    # TODO: Add some unit tests.
-    None
+    """
+    # TODO: this test currently needs connected device to pass
+    #       Need to remove that dependency before enabling the test
+    def test_report_result(self):
+        device_id = get_device_id()
+        camera_id = "1"
+        result_key = ItsSession.RESULT_KEY
+        results = {"scene0":{result_key:"PASS"},
+                   "scene1":{result_key:"PASS"},
+                   "scene2":{result_key:"PASS"},
+                   "scene3":{result_key:"PASS"},
+                   "sceneNotExist":{result_key:"FAIL"}}
+        report_result(device_id, camera_id, results)
+    """
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/apps/CameraITS/tools/run_all_tests.py b/apps/CameraITS/tools/run_all_tests.py
index 301ea73..678c35c 100644
--- a/apps/CameraITS/tools/run_all_tests.py
+++ b/apps/CameraITS/tools/run_all_tests.py
@@ -20,12 +20,20 @@
 import sys
 import textwrap
 import its.device
+from its.device import ItsSession
 
 def main():
     """Run all the automated tests, saving intermediate files, and producing
     a summary/report of the results.
 
     Script should be run from the top-level CameraITS directory.
+
+    Command line Arguments:
+        camera: the camera(s) to be tested. Use comma to separate multiple
+                camera Ids. Ex: "camera=0,1" or "camera=1"
+        scenes: the test scene(s) to be executed. Use comma to separate multiple
+                scenes. Ex: "scenes=scene0,scene1" or "scenes=0,1,sensor_fusion"
+                (sceneX can be abbreviated by X where X is a integer)
     """
 
     SKIP_RET_CODE = 101
@@ -49,9 +57,7 @@
         "sensor_fusion":[]
     }
 
-    # Get all the scene0 and scene1 tests, which can be run using the same
-    # physical setup.
-    scenes = ["scene0", "scene1", "scene2", "scene3", "scene4", "scene5"]
+    all_scenes = ["scene0", "scene1", "scene2", "scene3", "scene4", "scene5"]
 
     scene_req = {
         "scene0" : None,
@@ -74,12 +80,48 @@
     scene_extra_args = {
         "scene5" : ["doAF=False"]
     }
-    tests = []
-    for d in scenes:
-        tests += [(d,s[:-3],os.path.join("tests", d, s))
-                  for s in os.listdir(os.path.join("tests",d))
-                  if s[-3:] == ".py"]
-    tests.sort()
+
+    camera_ids = []
+    scenes = []
+    for s in sys.argv[1:]:
+        if s[:7] == "camera=" and len(s) > 7:
+            camera_ids = s[7:].split(',')
+        elif s[:7] == "scenes=" and len(s) > 7:
+            scenes = s[7:].split(',')
+
+    # Run through all scenes if user does not supply one
+    if not scenes:
+        scenes = all_scenes
+    else:
+        # Validate user input scene names
+        valid_scenes = True
+        temp_scenes = []
+        for s in scenes:
+            if s in all_scenes:
+                temp_scenes.append(s)
+            else:
+                try:
+                    # Try replace "X" to "sceneX"
+                    scene_num = int(s)
+                    scene_str = "scene" + s
+                    if scene_str not in all_scenes:
+                        valid_scenes = False
+                        break
+                    temp_scenes.append(scene_str)
+                except ValueError:
+                    valid_scenes = False
+                    break
+
+        if not valid_scenes:
+            print "Unknown scene specifiied:", s
+            assert(False)
+        scenes = temp_scenes
+
+    # Initialize test results
+    results = {}
+    result_key = ItsSession.RESULT_KEY
+    for s in all_scenes:
+        results[s] = {result_key: ItsSession.RESULT_NOT_EXECUTED}
 
     # Make output directories to hold the generated files.
     topdir = tempfile.mkdtemp()
@@ -89,11 +131,6 @@
     device_id_arg = "device=" + device_id
     print "Testing device " + device_id
 
-    camera_ids = []
-    for s in sys.argv[1:]:
-        if s[:7] == "camera=" and len(s) > 7:
-            camera_ids.append(s[7:])
-
     # user doesn't specify camera id, run through all cameras
     if not camera_ids:
         camera_ids_path = os.path.join(topdir, "camera_ids.txt")
@@ -107,7 +144,7 @@
             for line in f:
                 camera_ids.append(line.replace('\n', ''))
 
-    print "Running ITS on the following cameras:", camera_ids
+    print "Running ITS on camera: %s, scene %s" % (camera_ids, scenes)
 
     for camera_id in camera_ids:
         # Loop capturing images until user confirm test scene is correct
@@ -118,17 +155,18 @@
         for d in scenes:
             os.mkdir(os.path.join(topdir, camera_id, d))
 
-        print "Start running ITS on camera: ", camera_id
-        # Run each test, capturing stdout and stderr.
-        summary = "ITS test result summary for camera " + camera_id + "\n"
-        numpass = 0
-        numskip = 0
-        numnotmandatedfail = 0
-        numfail = 0
+        for scene in scenes:
+            tests = [(s[:-3],os.path.join("tests", scene, s))
+                     for s in os.listdir(os.path.join("tests",scene))
+                     if s[-3:] == ".py" and s[:4] == "test"]
+            tests.sort()
 
-        prev_scene = ""
-        for (scene,testname,testpath) in tests:
-            if scene != prev_scene and scene_req[scene] != None:
+            summary = "Cam" + camera_id + " " + scene + "\n"
+            numpass = 0
+            numskip = 0
+            num_not_mandated_fail = 0
+            numfail = 0
+            if scene_req[scene] != None:
                 out_path = os.path.join(topdir, camera_id, scene+".jpg")
                 out_arg = "out=" + out_path
                 scene_arg = "scene=" + scene_req[scene]
@@ -139,64 +177,71 @@
                         extra_args
                 retcode = subprocess.call(cmd,cwd=topdir)
                 assert(retcode == 0)
-                print "Start running tests for", scene
-            prev_scene = scene
-            cmd = ['python', os.path.join(os.getcwd(),testpath)] + \
-                  sys.argv[1:] + [camera_id_arg]
-            outdir = os.path.join(topdir,camera_id,scene)
-            outpath = os.path.join(outdir,testname+"_stdout.txt")
-            errpath = os.path.join(outdir,testname+"_stderr.txt")
-            t0 = time.time()
-            with open(outpath,"w") as fout, open(errpath,"w") as ferr:
-                retcode = subprocess.call(cmd,stderr=ferr,stdout=fout,cwd=outdir)
-            t1 = time.time()
+            print "Start running ITS on camera %s, %s" % (camera_id, scene)
 
-            if retcode == 0:
-                retstr = "PASS "
-                numpass += 1
-            elif retcode == SKIP_RET_CODE:
-                retstr = "SKIP "
-                numskip += 1
-            elif retcode != 0 and testname in NOT_YET_MANDATED[scene]:
-                retstr = "FAIL*"
-                numnotmandatedfail += 1
+            # Run each test, capturing stdout and stderr.
+            for (testname,testpath) in tests:
+                cmd = ['python', os.path.join(os.getcwd(),testpath)] + \
+                      sys.argv[1:] + [camera_id_arg]
+                outdir = os.path.join(topdir,camera_id,scene)
+                outpath = os.path.join(outdir,testname+"_stdout.txt")
+                errpath = os.path.join(outdir,testname+"_stderr.txt")
+                t0 = time.time()
+                with open(outpath,"w") as fout, open(errpath,"w") as ferr:
+                    retcode = subprocess.call(
+                            cmd,stderr=ferr,stdout=fout,cwd=outdir)
+                t1 = time.time()
+
+                test_failed = False
+                if retcode == 0:
+                    retstr = "PASS "
+                    numpass += 1
+                elif retcode == SKIP_RET_CODE:
+                    retstr = "SKIP "
+                    numskip += 1
+                elif retcode != 0 and testname in NOT_YET_MANDATED[scene]:
+                    retstr = "FAIL*"
+                    num_not_mandated_fail += 1
+                else:
+                    retstr = "FAIL "
+                    numfail += 1
+                    test_failed = True
+
+                msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0)
+                print msg
+                msg_short = "%s %s [%.1fs]" % (retstr, testname, t1-t0)
+                if test_failed:
+                    summary += msg_short + "\n"
+
+            if numskip > 0:
+                skipstr = ", %d test%s skipped" % (
+                        numskip, "s" if numskip > 1 else "")
             else:
-                retstr = "FAIL "
-                numfail += 1
+                skipstr = ""
 
-            msg = "%s %s/%s [%.1fs]" % (retstr, scene, testname, t1-t0)
-            print msg
-            summary += msg + "\n"
-            if retcode != 0 and retcode != SKIP_RET_CODE:
-                # Dump the stderr if the test fails
-                with open (errpath, "r") as error_file:
-                    errors = error_file.read()
-                    summary += errors + "\n"
+            test_result = "\n%d / %d tests passed (%.1f%%)%s" % (
+                    numpass + num_not_mandated_fail, len(tests) - numskip,
+                    100.0 * float(numpass + num_not_mandated_fail) /
+                            (len(tests) - numskip)
+                            if len(tests) != numskip else 100.0,
+                    skipstr)
+            print test_result
 
-        if numskip > 0:
-            skipstr = ", %d test%s skipped" % (numskip, "s" if numskip > 1 else "")
-        else:
-            skipstr = ""
+            if num_not_mandated_fail > 0:
+                msg = "(*) tests are not yet mandated"
+                print msg
 
-        test_result = "\n%d / %d tests passed (%.1f%%)%s" % (
-                numpass + numnotmandatedfail, len(tests) - numskip,
-                100.0 * float(numpass + numnotmandatedfail) / (len(tests) - numskip)
-                    if len(tests) != numskip else 100.0,
-                skipstr)
-        print test_result
-        summary += test_result + "\n"
+            summary_path = os.path.join(topdir, camera_id, scene, "summary.txt")
+            with open(summary_path, "w") as f:
+                f.write(summary)
 
-        if numnotmandatedfail > 0:
-            msg = "(*) tests are not yet mandated"
-            print msg
-            summary += msg + "\n"
+            passed = numfail == 0
+            results[scene][result_key] = (ItsSession.RESULT_PASS if passed
+                    else ItsSession.RESULT_FAIL)
+            results[scene][ItsSession.SUMMARY_KEY] = summary_path
 
-        result = numfail == 0
         print "Reporting ITS result to CtsVerifier"
-        summary_path = os.path.join(topdir, camera_id, "summary.txt")
-        with open(summary_path, "w") as f:
-            f.write(summary)
-        its.device.report_result(device_id, camera_id, result, summary_path)
+        its.device.report_result(device_id, camera_id, results)
 
     print "ITS tests finished. Please go back to CtsVerifier and proceed"
 
diff --git a/apps/CtsVerifier/res/layout/its_main.xml b/apps/CtsVerifier/res/layout/its_main.xml
index 2f5eade..26f15bb 100644
--- a/apps/CtsVerifier/res/layout/its_main.xml
+++ b/apps/CtsVerifier/res/layout/its_main.xml
@@ -21,4 +21,14 @@
 
     <include layout="@layout/pass_fail_buttons" />
 
+    <TextView
+        android:id="@+id/its_progress"
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:gravity="bottom"
+        android:padding="2dp"
+        android:scrollbars = "vertical"
+        android:text="@string/its_test_progress"
+        android:textSize="16sp" />
+
 </LinearLayout>
diff --git a/apps/CtsVerifier/res/values/strings.xml b/apps/CtsVerifier/res/values/strings.xml
index 52da8c4..2c81626 100644
--- a/apps/CtsVerifier/res/values/strings.xml
+++ b/apps/CtsVerifier/res/values/strings.xml
@@ -991,6 +991,10 @@
     </string>
     <string name="its_test_passed">All Camera ITS tests passed.  Pass button enabled!</string>
     <string name="its_test_failed">Some Camera ITS tests failed.</string>
+    <string name="its_version_mismatch">
+        CtsVerifier and ITS script version mismatch. Please update CtsVerifier and ITS script.
+    </string>
+    <string name="its_test_progress">ITS test progress will be shown here.</string>
 
     <!-- Strings for the Camera Flashlight test activity -->
     <string name="camera_flashlight_test">Camera Flashlight</string>
diff --git a/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsTestActivity.java b/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsTestActivity.java
index 0c39a9e..a8affcd 100644
--- a/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsTestActivity.java
+++ b/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsTestActivity.java
@@ -25,15 +25,21 @@
 import android.hardware.camera2.CameraCharacteristics;
 import android.hardware.camera2.CameraManager;
 import android.os.Bundle;
+import android.text.method.ScrollingMovementMethod;
 import android.util.Log;
 import android.view.WindowManager;
+import android.widget.TextView;
 import android.widget.Toast;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.HashMap;
-import java.util.Arrays;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
 import java.io.BufferedReader;
 import java.io.FileReader;
 import java.io.FileNotFoundException;
@@ -44,6 +50,8 @@
 import com.android.cts.verifier.PassFailButtons;
 import com.android.cts.verifier.R;
 
+import org.json.JSONArray;
+import org.json.JSONObject;
 
 /**
  * Test for Camera features that require that the camera be aimed at a specific test scene.
@@ -53,48 +61,216 @@
 public class ItsTestActivity extends PassFailButtons.Activity {
     private static final String TAG = "ItsTestActivity";
     private static final String EXTRA_CAMERA_ID = "camera.its.extra.CAMERA_ID";
-    private static final String EXTRA_SUCCESS = "camera.its.extra.SUCCESS";
-    private static final String EXTRA_SUMMARY = "camera.its.extra.SUMMARY";
+    private static final String EXTRA_RESULTS = "camera.its.extra.RESULTS";
+    private static final String EXTRA_VERSION = "camera.its.extra.VERSION";
+    private static final String CURRENT_VERSION = "1.0";
     private static final String ACTION_ITS_RESULT =
             "com.android.cts.verifier.camera.its.ACTION_ITS_RESULT";
 
-    class SuccessReceiver extends BroadcastReceiver {
+    private static final String RESULT_PASS = "PASS";
+    private static final String RESULT_FAIL = "FAIL";
+    private static final String RESULT_NOT_EXECUTED = "NOT_EXECUTED";
+    private static final Set<String> RESULT_VALUES = new HashSet<String>(
+            Arrays.asList(new String[] {RESULT_PASS, RESULT_FAIL, RESULT_NOT_EXECUTED}));
+    private static final int MAX_SUMMARY_LEN = 200;
+
+    private final ResultReceiver mResultsReceiver = new ResultReceiver();
+
+    // Initialized in onCreate
+    ArrayList<String> mNonLegacyCameraIds = null;
+
+    // TODO: cache the following in saved bundle
+    private Set<ResultKey> mAllScenes = null;
+    // (camera, scene) -> (pass, fail)
+    private final HashMap<ResultKey, Boolean> mExecutedScenes = new HashMap<>();
+    // map camera id to ITS summary report path
+    private final HashMap<ResultKey, String> mSummaryMap = new HashMap<>();
+
+    final class ResultKey {
+        public final String cameraId;
+        public final String sceneId;
+
+        public ResultKey(String cameraId, String sceneId) {
+            this.cameraId = cameraId;
+            this.sceneId = sceneId;
+        }
+
+        @Override
+        public boolean equals(final Object o) {
+            if (o == null) return false;
+            if (this == o) return true;
+            if (o instanceof ResultKey) {
+                final ResultKey other = (ResultKey) o;
+                return cameraId.equals(other.cameraId) && sceneId.equals(other.sceneId);
+            }
+            return false;
+        }
+
+        @Override
+        public int hashCode() {
+            int h = cameraId.hashCode();
+            h = ((h << 5) - h) ^ sceneId.hashCode();
+            return h;
+        }
+    }
+
+    private final Comparator<ResultKey> mComparator = new Comparator<ResultKey>() {
+        @Override
+        public int compare(ResultKey k1, ResultKey k2) {
+            if (k1.cameraId.equals(k2.cameraId))
+                return k1.sceneId.compareTo(k2.sceneId);
+            return k1.cameraId.compareTo(k2.cameraId);
+        }
+    };
+
+    class ResultReceiver extends BroadcastReceiver {
         @Override
         public void onReceive(Context context, Intent intent) {
             Log.i(TAG, "Received result for Camera ITS tests");
             if (ACTION_ITS_RESULT.equals(intent.getAction())) {
+                String version = intent.getStringExtra(EXTRA_VERSION);
+                if (version == null || !version.equals(CURRENT_VERSION)) {
+                    Log.e(TAG, "Its result version mismatch: expect " + CURRENT_VERSION +
+                            ", got " + ((version == null) ? "null" : version));
+                    ItsTestActivity.this.showToast(R.string.its_version_mismatch);
+                    return;
+                }
+
                 String cameraId = intent.getStringExtra(EXTRA_CAMERA_ID);
-                String result = intent.getStringExtra(EXTRA_SUCCESS);
-                String summaryPath = intent.getStringExtra(EXTRA_SUMMARY);
+                String results = intent.getStringExtra(EXTRA_RESULTS);
+                if (cameraId == null || results == null) {
+                    Log.e(TAG, "cameraId = " + ((cameraId == null) ? "null" : cameraId) +
+                            ", results = " + ((results == null) ? "null" : results));
+                    return;
+                }
+
                 if (!mNonLegacyCameraIds.contains(cameraId)) {
                     Log.e(TAG, "Unknown camera id " + cameraId + " reported to ITS");
                     return;
                 }
 
-                Log.i(TAG, "ITS summary path is: " + summaryPath);
-                mSummaryMap.put(cameraId, summaryPath);
-                // Create summary report
-                if (mSummaryMap.keySet().containsAll(mNonLegacyCameraIds)) {
+                try {
+                    /* Sample JSON results string
+                    {
+                       "scene0":{
+                          "result":"PASS",
+                          "summary":"/sdcard/cam0_scene0.txt"
+                       },
+                       "scene1":{
+                          "result":"NOT_EXECUTED"
+                       },
+                       "scene2":{
+                          "result":"FAIL",
+                          "summary":"/sdcard/cam0_scene2.txt"
+                       }
+                    }
+                    */
+                    JSONObject jsonResults = new JSONObject(results);
+                    Set<String> scenes = new HashSet<>();
+                    Iterator<String> keys = jsonResults.keys();
+                    while (keys.hasNext()) {
+                        scenes.add(keys.next());
+                    }
+                    boolean newScenes = false;
+                    if (mAllScenes == null) {
+                        mAllScenes = new TreeSet<>(mComparator);
+                        newScenes = true;
+                    } else { // See if scene lists changed
+                        for (String scene : scenes) {
+                            if (!mAllScenes.contains(new ResultKey(cameraId, scene))) {
+                                // Scene list changed. Cleanup previous test results
+                                newScenes = true;
+                                break;
+                            }
+                        }
+                        for (ResultKey k : mAllScenes) {
+                            if (!scenes.contains(k.sceneId)) {
+                                newScenes = true;
+                                break;
+                            }
+                        }
+                    }
+                    if (newScenes) {
+                        mExecutedScenes.clear();
+                        mAllScenes.clear();
+                        for (String scene : scenes) {
+                            for (String c : mNonLegacyCameraIds) {
+                                mAllScenes.add(new ResultKey(c, scene));
+                            }
+                        }
+                    }
+
+                    // Update test execution results
+                    for (String scene : scenes) {
+                        JSONObject sceneResult = jsonResults.getJSONObject(scene);
+                        String result = sceneResult.getString("result");
+                        if (result == null) {
+                            Log.e(TAG, "Result for " + scene + " is null");
+                            return;
+                        }
+                        Log.i(TAG, "ITS camera" + cameraId + " " + scene + ": result:" + result);
+                        if (!RESULT_VALUES.contains(result)) {
+                            Log.e(TAG, "Unknown result for " + scene + ": " + result);
+                            return;
+                        }
+                        ResultKey key = new ResultKey(cameraId, scene);
+                        if (result.equals(RESULT_PASS) || result.equals(RESULT_FAIL)) {
+                            boolean pass = result.equals(RESULT_PASS);
+                            mExecutedScenes.put(key, pass);
+                            String summary = sceneResult.optString("summary");
+                            if (!summary.equals("")) {
+                                mSummaryMap.put(key, summary);
+                            }
+                        } // do nothing for NOT_EXECUTED scenes
+                    }
+                } catch (org.json.JSONException e) {
+                    Log.e(TAG, "Error reading json result string:" + results , e);
+                    return;
+                }
+
+                // Set summary if all scenes reported
+                if (mSummaryMap.keySet().containsAll(mAllScenes)) {
                     StringBuilder summary = new StringBuilder();
-                    for (String id : mNonLegacyCameraIds) {
-                        String path = mSummaryMap.get(id);
+                    for (String path : mSummaryMap.values()) {
                         appendFileContentToSummary(summary, path);
                     }
+                    if (summary.length() > MAX_SUMMARY_LEN) {
+                        Log.w(TAG, "ITS summary report too long: len: " + summary.length());
+                    }
                     ItsTestActivity.this.getReportLog().setSummary(
                             summary.toString(), 1.0, ResultType.NEUTRAL, ResultUnit.NONE);
                 }
-                boolean pass = result.equals("True");
-                if(pass) {
-                    Log.i(TAG, "Received Camera " + cameraId + " ITS SUCCESS from host.");
-                    mITSPassedCameraIds.add(cameraId);
-                    if (mNonLegacyCameraIds != null && mNonLegacyCameraIds.size() != 0 &&
-                            mITSPassedCameraIds.containsAll(mNonLegacyCameraIds)) {
-                        ItsTestActivity.this.showToast(R.string.its_test_passed);
-                        ItsTestActivity.this.getPassButton().setEnabled(true);
+
+                // Display current progress
+                StringBuilder progress = new StringBuilder();
+                for (ResultKey k : mAllScenes) {
+                    String status = RESULT_NOT_EXECUTED;
+                    if (mExecutedScenes.containsKey(k)) {
+                        status = mExecutedScenes.get(k) ? RESULT_PASS : RESULT_FAIL;
                     }
+                    progress.append(String.format("Cam %s, %s: %s\n",
+                            k.cameraId, k.sceneId, status));
+                }
+                TextView progressView = (TextView) findViewById(R.id.its_progress);
+                progressView.setMovementMethod(new ScrollingMovementMethod());
+                progressView.setText(progress.toString());
+
+
+                // Enable pass button if all scenes pass
+                boolean allScenesPassed = true;
+                for (ResultKey k : mAllScenes) {
+                    Boolean pass = mExecutedScenes.get(k);
+                    if (pass == null || pass == false) {
+                        allScenesPassed = false;
+                        break;
+                    }
+                }
+                if (allScenesPassed) {
+                    // Enable pass button
+                    ItsTestActivity.this.showToast(R.string.its_test_passed);
+                    ItsTestActivity.this.getPassButton().setEnabled(true);
                 } else {
-                    Log.i(TAG, "Received Camera " + cameraId + " ITS FAILURE from host.");
-                    ItsTestActivity.this.showToast(R.string.its_test_failed);
+                    ItsTestActivity.this.getPassButton().setEnabled(false);
                 }
             }
         }
@@ -127,12 +303,6 @@
         }
     }
 
-    private final SuccessReceiver mSuccessReceiver = new SuccessReceiver();
-    private final HashSet<String> mITSPassedCameraIds = new HashSet<>();
-    // map camera id to ITS summary report path
-    private final HashMap<String, String> mSummaryMap = new HashMap<>();
-    ArrayList<String> mNonLegacyCameraIds = null;
-
     @Override
     protected void onCreate(Bundle savedInstanceState) {
         super.onCreate(savedInstanceState);
@@ -168,6 +338,7 @@
                     "Received error from camera service while checking device capabilities: "
                             + e, Toast.LENGTH_SHORT).show();
         }
+
         getPassButton().setEnabled(false);
     }
 
@@ -180,7 +351,7 @@
         } else {
             Log.d(TAG, "register ITS result receiver");
             IntentFilter filter = new IntentFilter(ACTION_ITS_RESULT);
-            registerReceiver(mSuccessReceiver, filter);
+            registerReceiver(mResultsReceiver, filter);
         }
     }
 
@@ -188,7 +359,7 @@
     protected void onPause() {
         super.onPause();
         Log.d(TAG, "unregister ITS result receiver");
-        unregisterReceiver(mSuccessReceiver);
+        unregisterReceiver(mResultsReceiver);
     }
 
     @Override