CameraITS: Add stats image format.
Bug: 23980354
Change-Id: I8414093d77ab2be9f335bde85ec06ab86df62fd9
diff --git a/apps/CameraITS/pymodules/its/device.py b/apps/CameraITS/pymodules/its/device.py
index 756f959..835a4a4 100644
--- a/apps/CameraITS/pymodules/its/device.py
+++ b/apps/CameraITS/pymodules/its/device.py
@@ -448,8 +448,8 @@
The out_surfaces field can specify the width(s), height(s), and
format(s) of the captured image. The formats may be "yuv", "jpeg",
- "dng", "raw", "raw10", or "raw12". The default is a YUV420 frame ("yuv")
- corresponding to a full sensor frame.
+ "dng", "raw", "raw10", "raw12", or "rawStats". The default is a YUV420
+ frame ("yuv") corresponding to a full sensor frame.
Note that one or more surfaces can be specified, allowing a capture to
request images back in multiple formats (e.g.) raw+yuv, raw+jpeg,
@@ -536,6 +536,25 @@
yuv_caps = do_capture( [req1,req2], yuv_fmt )
yuv_caps, raw_caps = do_capture( [req1,req2], [yuv_fmt,raw_fmt] )
+ The "rawStats" format processes the raw image and returns a new image
+ of statistics from the raw image. The format takes additional keys,
+ "gridWidth" and "gridHeight" which are size of grid cells in a 2D grid
+ of the raw image. For each grid cell, the mean and variance of each raw
+ channel is computed, and the do_capture call returns two 4-element float
+ images of dimensions (rawWidth / gridWidth, rawHeight / gridHeight),
+ concatenated back-to-back, where the first iamge contains the 4-channel
+ means and the second contains the 4-channel variances.
+
+ For the rawStats format, if the gridWidth is not provided then the raw
+ image width is used as the default, and similarly for gridHeight. With
+ this, the following is an example of a output description that computes
+ the mean and variance across each image row:
+
+ {
+ "gridHeight": 1,
+ "format": "rawStats"
+ }
+
Args:
cap_request: The Python dict/list specifying the capture(s), which
will be converted to JSON and sent to the device.
@@ -550,7 +569,8 @@
* data: the image data as a numpy array of bytes.
* width: the width of the captured image.
* height: the height of the captured image.
- * format: image the format, in ["yuv","jpeg","raw","raw10","dng"].
+ * format: image the format, in [
+ "yuv","jpeg","raw","raw10","raw12","rawStats","dng"].
* metadata: the capture result object (Python dictionary).
"""
cmd = {}
@@ -577,9 +597,13 @@
nsurf = 1 if out_surfaces is None else len(cmd["outputSurfaces"])
if len(formats) > len(set(formats)):
raise its.error.Error('Duplicate format requested')
- if "dng" in formats and "raw" in formats or \
- "dng" in formats and "raw10" in formats or \
- "raw" in formats and "raw10" in formats:
+ raw_formats = 0;
+ raw_formats += 1 if "dng" in formats else 0
+ raw_formats += 1 if "raw" in formats else 0
+ raw_formats += 1 if "raw10" in formats else 0
+ raw_formats += 1 if "raw12" in formats else 0
+ raw_formats += 1 if "rawStats" in formats else 0
+ if raw_formats > 1:
raise its.error.Error('Different raw formats not supported')
# Detect long exposure time and set timeout accordingly
@@ -603,14 +627,16 @@
# the burst, however individual images of different formats can come
# out in any order for that capture.
nbufs = 0
- bufs = {"yuv":[], "raw":[], "raw10":[], "dng":[], "jpeg":[]}
+ bufs = {"yuv":[], "raw":[], "raw10":[], "raw12":[],
+ "rawStats":[], "dng":[], "jpeg":[]}
mds = []
widths = None
heights = None
while nbufs < ncap*nsurf or len(mds) < ncap:
jsonObj,buf = self.__read_response_from_socket()
if jsonObj['tag'] in ['jpegImage', 'yuvImage', 'rawImage', \
- 'raw10Image', 'dngImage'] and buf is not None:
+ 'raw10Image', 'raw12Image', 'rawStatsImage', 'dngImage'] \
+ and buf is not None:
fmt = jsonObj['tag'][:-5]
bufs[fmt].append(buf)
nbufs += 1
diff --git a/apps/CameraITS/pymodules/its/image.py b/apps/CameraITS/pymodules/its/image.py
index ea01a3e..3575221 100644
--- a/apps/CameraITS/pymodules/its/image.py
+++ b/apps/CameraITS/pymodules/its/image.py
@@ -81,6 +81,25 @@
else:
raise its.error.Error('Invalid format %s' % (cap["format"]))
+def unpack_rawstats_capture(cap):
+ """Unpack a rawStats capture to the mean and variance images.
+
+ Args:
+ cap: A capture object as returned by its.device.do_capture.
+
+ Returns:
+ Tuple (mean_image var_image) of float-4 images, with non-normalized
+ pixel values computed from the RAW16 images on the device
+ """
+ assert(cap["format"] == "rawStats")
+ w = cap["width"]
+ h = cap["height"]
+ img = numpy.ndarray(shape=(2*h*w*4,), dtype='<f', buffer=cap["data"])
+ analysis_image = img.reshape(2,h,w,4)
+ mean_image = analysis_image[0,:,:,:].reshape(h,w,4)
+ var_image = analysis_image[1,:,:,:].reshape(h,w,4)
+ return mean_image, var_image
+
def unpack_raw10_capture(cap, props):
"""Unpack a raw-10 capture to a raw-16 capture.
diff --git a/apps/CameraITS/tests/inprog/test_rawstats.py b/apps/CameraITS/tests/inprog/test_rawstats.py
new file mode 100644
index 0000000..8083f0b
--- /dev/null
+++ b/apps/CameraITS/tests/inprog/test_rawstats.py
@@ -0,0 +1,48 @@
+# Copyright 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import its.image
+import its.caps
+import its.device
+import its.objects
+import its.target
+import os.path
+import math
+
+def main():
+ """Test capturing some rawstats data.
+ """
+ NAME = os.path.basename(__file__).split(".")[0]
+
+ with its.device.ItsSession() as cam:
+
+ cam.do_3a(do_af=False);
+ req = its.objects.auto_capture_request()
+
+ for (gw,gh) in [(16,16)]:#,(4080,1)]:
+ cap = cam.do_capture(req,
+ {"format":"rawStats","gridWidth":gw,"gridHeight":gh})
+ mean_image, var_image = its.image.unpack_rawstats_capture(cap)
+
+ if gw > 1 and gh > 1:
+ h,w,_ = mean_image.shape
+ for ch in range(4):
+ m = mean_image[:,:,ch].reshape(h,w,1)/1023.0
+ v = var_image[:,:,ch].reshape(h,w,1)
+ its.image.write_image(m, "%s_mean_ch%d.jpg" % (NAME,ch), True)
+ its.image.write_image(v, "%s_var_ch%d.jpg" % (NAME,ch), True)
+
+if __name__ == '__main__':
+ main()
+
diff --git a/apps/CtsVerifier/jni/verifier/Android.mk b/apps/CtsVerifier/jni/verifier/Android.mk
index da4687d..6bdf71c 100644
--- a/apps/CtsVerifier/jni/verifier/Android.mk
+++ b/apps/CtsVerifier/jni/verifier/Android.mk
@@ -25,8 +25,11 @@
LOCAL_SRC_FILES := \
CtsVerifierJniOnLoad.cpp \
- com_android_cts_verifier_os_FileUtils.cpp
+ com_android_cts_verifier_camera_StatsImage.cpp \
+ com_android_cts_verifier_os_FileUtils.cpp
LOCAL_C_INCLUDES := $(JNI_H_INCLUDE)
+LOCAL_LDLIBS := -llog
+
include $(BUILD_SHARED_LIBRARY)
diff --git a/apps/CtsVerifier/jni/verifier/CtsVerifierJniOnLoad.cpp b/apps/CtsVerifier/jni/verifier/CtsVerifierJniOnLoad.cpp
index 81e5690..399275b 100644
--- a/apps/CtsVerifier/jni/verifier/CtsVerifierJniOnLoad.cpp
+++ b/apps/CtsVerifier/jni/verifier/CtsVerifierJniOnLoad.cpp
@@ -1,4 +1,4 @@
-/*
+/*
* Copyright (C) 2010 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,6 +18,7 @@
#include <stdio.h>
extern int register_com_android_cts_verifier_os_FileUtils(JNIEnv*);
+extern int register_com_android_cts_verifier_camera_its_StatsImage(JNIEnv*);
jint JNI_OnLoad(JavaVM *vm, void *reserved) {
JNIEnv *env = NULL;
@@ -30,5 +31,9 @@
return JNI_ERR;
}
+ if (register_com_android_cts_verifier_camera_its_StatsImage(env)) {
+ return JNI_ERR;
+ }
+
return JNI_VERSION_1_4;
}
diff --git a/apps/CtsVerifier/jni/verifier/com_android_cts_verifier_camera_StatsImage.cpp b/apps/CtsVerifier/jni/verifier/com_android_cts_verifier_camera_StatsImage.cpp
new file mode 100644
index 0000000..16dff85
--- /dev/null
+++ b/apps/CtsVerifier/jni/verifier/com_android_cts_verifier_camera_StatsImage.cpp
@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "ITS-StatsImage-JNI"
+// #define LOG_NDEBUG 0
+#include <android/log.h>
+#include <utils/Log.h>
+
+#include <jni.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <inttypes.h>
+#include <string.h>
+
+jfloatArray com_android_cts_verifier_camera_its_computeStatsImage(JNIEnv* env, jobject thiz,
+ jbyteArray img, jint width, jint height, jint gridWidth, jint gridHeight)
+{
+ int bufSize = (int)(env->GetArrayLength(img));
+ unsigned char *buf = (unsigned char*)env->GetByteArrayElements(img, /*is_copy*/NULL);
+
+ // Size of the raw image.
+ const int w = width;
+ const int h = height;
+ // Size of each grid cell.
+ const int gw = gridWidth;
+ const int gh = gridHeight;
+ // Number of grid cells (rounding down to full cells only at right+bottom edges).
+ const int ngx = w / gw;
+ const int ngy = h / gh;
+
+ float *mean = new float[ngy*ngx*4];
+ float *var = new float[ngy*ngx*4];
+ for (int gy = 0; gy < ngy; gy++) {
+ for (int gx = 0; gx < ngx; gx++) {
+ float sum[4] = {0};
+ float sumSq[4] = {0};
+ int count[4] = {0};
+ for (int y = gy*gh; y < (gy+1)*gh; y++) {
+ int chnOffset = (y & 0x1) * 2;
+ unsigned char *pbuf = buf + 2*y*w + 2*gx*gw;
+ for (int x = gx*gw; x < (gx+1)*gw; x++) {
+ // input is RAW16
+ int byte0 = *pbuf++;
+ int byte1 = *pbuf++;
+ int pixelValue = (byte1 << 8) | byte0;
+ int ch = chnOffset + (x & 1);
+ sum[ch] += pixelValue;
+ sumSq[ch] += pixelValue * pixelValue;
+ count[ch] += 1;
+ }
+ }
+ for (int ch = 0; ch < 4; ch++) {
+ float m = (float)sum[ch] / count[ch];
+ float mSq = (float)sumSq[ch] / count[ch];
+ mean[gy*ngx*4 + gx*4 + ch] = m;
+ var[gy*ngx*4 + gx*4 + ch] = mSq - m*m;
+ }
+ }
+ }
+
+ jfloatArray ret = env->NewFloatArray(ngx*ngy*4*2);
+ env->SetFloatArrayRegion(ret, 0, ngx*ngy*4, (float*)mean);
+ env->SetFloatArrayRegion(ret, ngx*ngy*4, ngx*ngy*4, (float*)var);
+ delete [] mean;
+ delete [] var;
+ return ret;
+}
+
+static JNINativeMethod gMethods[] = {
+ { "computeStatsImage", "([BIIII)[F",
+ (void *) com_android_cts_verifier_camera_its_computeStatsImage },
+};
+
+int register_com_android_cts_verifier_camera_its_StatsImage(JNIEnv* env)
+{
+ jclass clazz = env->FindClass("com/android/cts/verifier/camera/its/StatsImage");
+
+ return env->RegisterNatives(clazz, gMethods,
+ sizeof(gMethods) / sizeof(JNINativeMethod));
+}
diff --git a/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsService.java b/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsService.java
index 27f8c28..e3ff74b 100644
--- a/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsService.java
+++ b/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/ItsService.java
@@ -39,12 +39,14 @@
import android.media.Image;
import android.media.ImageReader;
import android.media.ImageWriter;
+import android.media.Image.Plane;
import android.net.Uri;
import android.os.ConditionVariable;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.IBinder;
import android.os.Message;
+import android.os.SystemClock;
import android.os.Vibrator;
import android.util.Log;
import android.util.Rational;
@@ -56,6 +58,8 @@
import com.android.ex.camera2.blocking.BlockingStateCallback;
import com.android.ex.camera2.blocking.BlockingSessionCallback;
+import com.android.cts.verifier.camera.its.StatsImage;
+
import org.json.JSONArray;
import org.json.JSONObject;
@@ -70,6 +74,8 @@
import java.net.ServerSocket;
import java.net.Socket;
import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.FloatBuffer;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.util.ArrayList;
@@ -154,6 +160,9 @@
private AtomicInteger mCountYuv = new AtomicInteger();
private AtomicInteger mCountCapRes = new AtomicInteger();
private boolean mCaptureRawIsDng;
+ private boolean mCaptureRawIsStats;
+ private int mCaptureStatsGridWidth;
+ private int mCaptureStatsGridHeight;
private CaptureResult mCaptureResults[] = null;
private volatile ConditionVariable mInterlock3A = new ConditionVariable(true);
@@ -404,7 +413,7 @@
continue;
}
if (b.hasArray()) {
- mOpenSocket.getOutputStream().write(b.array());
+ mOpenSocket.getOutputStream().write(b.array(), 0, b.capacity());
} else {
byte[] barray = new byte[b.capacity()];
b.get(barray);
@@ -665,7 +674,16 @@
jsonSurface.put("height", readers[i].getHeight());
int format = readers[i].getImageFormat();
if (format == ImageFormat.RAW_SENSOR) {
- jsonSurface.put("format", "raw");
+ if (mCaptureRawIsStats) {
+ jsonSurface.put("format", "rawStats");
+ jsonSurface.put("width", readers[i].getWidth()/mCaptureStatsGridWidth);
+ jsonSurface.put("height",
+ readers[i].getHeight()/mCaptureStatsGridHeight);
+ } else if (mCaptureRawIsDng) {
+ jsonSurface.put("format", "dng");
+ } else {
+ jsonSurface.put("format", "raw");
+ }
} else if (format == ImageFormat.RAW10) {
jsonSurface.put("format", "raw10");
} else if (format == ImageFormat.RAW12) {
@@ -1068,6 +1086,12 @@
outputFormats[i] = ImageFormat.RAW_SENSOR;
sizes = ItsUtils.getRaw16OutputSizes(mCameraCharacteristics);
mCaptureRawIsDng = true;
+ } else if ("rawStats".equals(sformat)) {
+ outputFormats[i] = ImageFormat.RAW_SENSOR;
+ sizes = ItsUtils.getRaw16OutputSizes(mCameraCharacteristics);
+ mCaptureRawIsStats = true;
+ mCaptureStatsGridWidth = surfaceObj.optInt("gridWidth");
+ mCaptureStatsGridHeight = surfaceObj.optInt("gridHeight");
} else {
throw new ItsException("Unsupported format: " + sformat);
}
@@ -1086,6 +1110,14 @@
if (height <= 0) {
height = ItsUtils.getMaxSize(sizes).getHeight();
}
+ if (mCaptureStatsGridWidth <= 0) {
+ mCaptureStatsGridWidth = width;
+ }
+ if (mCaptureStatsGridHeight <= 0) {
+ mCaptureStatsGridHeight = height;
+ }
+
+ // TODO: Crop to the active array in the stats image analysis.
outputSizes[i] = new Size(width, height);
}
@@ -1122,6 +1154,7 @@
mCountRaw12.set(0);
mCountCapRes.set(0);
mCaptureRawIsDng = false;
+ mCaptureRawIsStats = false;
mCaptureResults = new CaptureResult[requests.size()];
JSONArray jsonOutputSpecs = ItsUtils.getOutputSpecs(params);
@@ -1235,6 +1268,7 @@
mCountRaw12.set(0);
mCountCapRes.set(0);
mCaptureRawIsDng = false;
+ mCaptureRawIsStats = false;
try {
// Parse the JSON to get the list of capture requests.
@@ -1427,8 +1461,28 @@
int count = mCountRawOrDng.getAndIncrement();
if (! mCaptureRawIsDng) {
byte[] img = ItsUtils.getDataFromImage(capture);
- ByteBuffer buf = ByteBuffer.wrap(img);
- mSocketRunnableObj.sendResponseCaptureBuffer("rawImage", buf);
+ if (! mCaptureRawIsStats) {
+ ByteBuffer buf = ByteBuffer.wrap(img);
+ mSocketRunnableObj.sendResponseCaptureBuffer("rawImage", buf);
+ } else {
+ // Compute the requested stats on the raw frame, and return the results
+ // in a new "stats image".
+ long startTimeMs = SystemClock.elapsedRealtime();
+ int w = capture.getWidth();
+ int h = capture.getHeight();
+ int gw = mCaptureStatsGridWidth;
+ int gh = mCaptureStatsGridHeight;
+ float[] stats = StatsImage.computeStatsImage(img, w, h, gw, gh);
+ long endTimeMs = SystemClock.elapsedRealtime();
+ Log.e(TAG, "Raw stats computation takes " + (endTimeMs - startTimeMs) + " ms");
+
+ ByteBuffer bBuf = ByteBuffer.allocateDirect(stats.length * 4);
+ bBuf.order(ByteOrder.nativeOrder());
+ FloatBuffer fBuf = bBuf.asFloatBuffer();
+ fBuf.put(stats);
+ fBuf.position(0);
+ mSocketRunnableObj.sendResponseCaptureBuffer("rawStatsImage", bBuf);
+ }
} else {
// Wait until the corresponding capture result is ready, up to a timeout.
long t0 = android.os.SystemClock.elapsedRealtime();
diff --git a/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/StatsImage.java b/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/StatsImage.java
new file mode 100644
index 0000000..037177c
--- /dev/null
+++ b/apps/CtsVerifier/src/com/android/cts/verifier/camera/its/StatsImage.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.cts.verifier.camera.its;
+
+import android.util.Log;
+
+public class StatsImage {
+
+ static {
+ try {
+ System.loadLibrary("ctsverifier_jni");
+ } catch (UnsatisfiedLinkError e) {
+ Log.e("StatsImage", "Error loading cts verifier JNI library");
+ e.printStackTrace();
+ }
+ }
+
+ public native static float[] computeStatsImage(
+ byte[] img, int width, int height, int gridW, int gridH);
+
+}
