RVCVXCheck CTS video recording bugfixes
Fix the following:
* Video recorded do not have right aspect ratio on Sprout devices.
* Video timestamp is estimated from fps ratings, which may be
off by 5% on some devices.
* Stretched video preview display.
* Rotated video preview in some devices.
* CPU goes to sleep during long analaysis.
Related bug(s):
b/23297751
b/23911906
Change-Id: I98a9b49e1f2587f1d0448ea9d1c2e229fdaee29b
diff --git a/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVCameraPreview.java b/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVCameraPreview.java
index a5b58f6..fa89b71 100644
--- a/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVCameraPreview.java
+++ b/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVCameraPreview.java
@@ -23,9 +23,9 @@
import android.util.Log;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
+import android.view.ViewGroup;
import java.io.IOException;
-import java.util.List;
/** Camera preview class */
public class RVCVCameraPreview extends SurfaceView implements SurfaceHolder.Callback {
@@ -34,15 +34,16 @@
private SurfaceHolder mHolder;
private Camera mCamera;
+ private float mAspect;
+ private int mRotation;
/**
* Constructor
* @param context Activity context
- * @param camera Camera object to be previewed
*/
- public RVCVCameraPreview(Context context, Camera camera) {
+ public RVCVCameraPreview(Context context) {
super(context);
- mCamera = camera;
+ mCamera = null;
initSurface();
}
@@ -55,8 +56,10 @@
super(context, attrs);
}
- public void init(Camera camera) {
+ public void init(Camera camera, float aspectRatio, int rotation) {
this.mCamera = camera;
+ mAspect = aspectRatio;
+ mRotation = rotation;
initSurface();
}
@@ -86,6 +89,20 @@
try {
mCamera.setPreviewDisplay(holder);
mCamera.startPreview();
+ int v_height = getHeight();
+ int v_width = getWidth();
+ ViewGroup.LayoutParams layout = getLayoutParams();
+ if ( (float)v_height/v_width >
+ mAspect) {
+ layout.height = (int)(v_width * mAspect);
+ layout.width = v_width;
+ }else {
+ layout.width = (int)(v_height / mAspect);
+ layout.height = v_height;
+ }
+ Log.d(TAG, String.format("Layout (%d, %d) -> (%d, %d)", v_width, v_height,
+ layout.width, layout.height));
+ setLayoutParams(layout);
} catch (IOException e) {
if (LOCAL_LOGD) Log.d(TAG, "Error when starting camera preview: " + e.getMessage());
}
@@ -111,8 +128,7 @@
// stop preview before making changes
mCamera.stopPreview();
- // the activity using this view is locked to this orientation, so hard code is fine
- mCamera.setDisplayOrientation(90);
+ mCamera.setDisplayOrientation(mRotation);
//do the same as if it is created again
surfaceCreated(holder);
diff --git a/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVRecordActivity.java b/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVRecordActivity.java
index f90b27c..be5ec52 100644
--- a/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVRecordActivity.java
+++ b/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVRecordActivity.java
@@ -46,6 +46,7 @@
import java.io.OutputStreamWriter;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.List;
// ----------------------------------------------------------------------
@@ -67,7 +68,7 @@
private VideoRecorder mVideoRecorder;
private RVSensorLogger mRVSensorLogger;
private CoverageManager mCoverManager;
- private CameraPreviewer mPreviewer;
+ private CameraContext mCameraContext;
public static final int AXIS_NONE = 0;
public static final int AXIS_ALL = SensorManager.AXIS_X +
@@ -99,7 +100,7 @@
super.onPause();
mController.quit();
- mPreviewer.end();
+ mCameraContext.end();
endSoundPool();
}
@@ -128,8 +129,8 @@
*
*/
private void init() {
- mPreviewer = new CameraPreviewer();
- mPreviewer.init();
+ mCameraContext = new CameraContext();
+ mCameraContext.init();
mCoverManager = new CoverageManager();
mIndicatorView.setDataProvider(
@@ -140,7 +141,7 @@
initSoundPool();
mRVSensorLogger = new RVSensorLogger(this);
- mVideoRecorder = new VideoRecorder(mPreviewer.getCamera());
+ mVideoRecorder = new VideoRecorder(mCameraContext.getCamera(), mCameraContext.getProfile());
if (LOG_RAW_SENSORS) {
mRawSensorLogger = new RawSensorLogger(mRecordDir);
@@ -173,7 +174,8 @@
// X and Y
final String axisName = "YXZ";
- message("Manipulate the device in " + axisName.charAt(axis-1) + " axis (as illustrated) about the pattern.");
+ message("Manipulate the device in " + axisName.charAt(axis - 1) +
+ " axis (as illustrated) about the pattern.");
}
/**
@@ -250,20 +252,28 @@
* Start the sensor recording
*/
public void startRecordSensor() {
- mRVSensorLogger.init();
- if (LOG_RAW_SENSORS) {
- mRawSensorLogger.init();
- }
+ runOnUiThread(new Runnable() {
+ public void run() {
+ mRVSensorLogger.init();
+ if (LOG_RAW_SENSORS) {
+ mRawSensorLogger.init();
+ }
+ }
+ });
}
/**
* Stop the sensor recording
*/
public void stopRecordSensor() {
- mRVSensorLogger.end();
- if (LOG_RAW_SENSORS) {
- mRawSensorLogger.end();
- }
+ runOnUiThread(new Runnable() {
+ public void run() {
+ mRVSensorLogger.end();
+ if (LOG_RAW_SENSORS) {
+ mRawSensorLogger.end();
+ }
+ }
+ });
}
/**
@@ -365,20 +375,93 @@
/**
* Camera preview control class
*/
- class CameraPreviewer {
+ class CameraContext {
private Camera mCamera;
+ private CamcorderProfile mProfile;
+ private Camera.CameraInfo mCameraInfo;
- CameraPreviewer() {
+ private int [] mPreferredProfiles = {
+ CamcorderProfile.QUALITY_480P, // smaller -> faster
+ CamcorderProfile.QUALITY_720P,
+ CamcorderProfile.QUALITY_1080P,
+ CamcorderProfile.QUALITY_HIGH // existence guaranteed
+ };
+
+ CameraContext() {
try {
- mCamera = Camera.open(); // attempt to get a default Camera instance
+ mCamera = Camera.open(); // attempt to get a default Camera instance (0)
+ mProfile = null;
+ if (mCamera != null) {
+ mCameraInfo = new Camera.CameraInfo();
+ Camera.getCameraInfo(0, mCameraInfo);
+ setupCamera();
+ }
}
- catch (Exception e) {
+ catch (Exception e){
// Camera is not available (in use or does not exist)
Log.e(TAG, "Cannot obtain Camera!");
}
}
/**
+ * Find a preferred camera profile and set preview and picture size property accordingly.
+ */
+ void setupCamera() {
+ CamcorderProfile profile = null;
+ Camera.Parameters param = mCamera.getParameters();
+ List<Camera.Size> pre_sz = param.getSupportedPreviewSizes();
+ List<Camera.Size> pic_sz = param.getSupportedPictureSizes();
+
+ for (int i : mPreferredProfiles) {
+ if (CamcorderProfile.hasProfile(i)) {
+ profile = CamcorderProfile.get(i);
+
+ int valid = 0;
+ for (Camera.Size j : pre_sz) {
+ if (j.width == profile.videoFrameWidth &&
+ j.height == profile.videoFrameHeight) {
+ ++valid;
+ break;
+ }
+ }
+ for (Camera.Size j : pic_sz) {
+ if (j.width == profile.videoFrameWidth &&
+ j.height == profile.videoFrameHeight) {
+ ++valid;
+ break;
+ }
+ }
+ if (valid == 2) {
+ param.setPreviewSize(profile.videoFrameWidth, profile.videoFrameHeight);
+ param.setPictureSize(profile.videoFrameWidth, profile.videoFrameHeight);
+ mCamera.setParameters(param);
+ break;
+ } else {
+ profile = null;
+ }
+ }
+ }
+ if (profile != null) {
+ float fovW = param.getHorizontalViewAngle();
+ float fovH = param.getVerticalViewAngle();
+ writeVideoMetaInfo(profile.videoFrameWidth, profile.videoFrameHeight,
+ profile.videoFrameRate, fovW, fovH);
+ } else {
+ Log.e(TAG, "Cannot find a proper video profile");
+ }
+ mProfile = profile;
+
+ }
+
+
+ /**
+ * Get sensor information of the camera being used
+ */
+ public Camera.CameraInfo getCameraInfo() {
+ return mCameraInfo;
+ }
+
+ /**
* Get the camera to be previewed
* @return Reference to Camera used
*/
@@ -387,12 +470,20 @@
}
/**
+ * Get the camera profile to be used
+ * @return Reference to Camera profile
+ */
+ public CamcorderProfile getProfile() {
+ return mProfile;
+ }
+
+ /**
* Setup the camera
*/
public void init() {
if (mCamera != null) {
double alpha = mCamera.getParameters().getHorizontalViewAngle()*Math.PI/180.0;
- int width = 1920;
+ int width = mProfile.videoFrameWidth;
double fx = width/2/Math.tan(alpha/2.0);
if (LOCAL_LOGV) Log.v(TAG, "View angle="
@@ -400,7 +491,9 @@
RVCVCameraPreview cameraPreview =
(RVCVCameraPreview) findViewById(R.id.cam_preview);
- cameraPreview.init(mCamera);
+ cameraPreview.init(mCamera,
+ (float)mProfile.videoFrameWidth/mProfile.videoFrameHeight,
+ mCameraInfo.orientation);
} else {
message("Cannot open camera!");
finish();
@@ -466,26 +559,22 @@
class VideoRecorder
{
private MediaRecorder mRecorder;
+ private CamcorderProfile mProfile;
private Camera mCamera;
private boolean mRunning = false;
- private int [] mPreferredProfiles = { CamcorderProfile.QUALITY_480P, // smaller -> faster
- CamcorderProfile.QUALITY_720P,
- CamcorderProfile.QUALITY_1080P,
- CamcorderProfile.QUALITY_HIGH // existence guaranteed
- };
-
-
- VideoRecorder(Camera camera) {
+ VideoRecorder(Camera camera, CamcorderProfile profile){
mCamera = camera;
+ mProfile = profile;
}
/**
* Initialize and start recording
*/
public void init() {
- float fovW = mCamera.getParameters().getHorizontalViewAngle();
- float fovH = mCamera.getParameters().getVerticalViewAngle();
+ if (mCamera == null || mProfile ==null){
+ return;
+ }
mRecorder = new MediaRecorder();
mCamera.unlock();
@@ -494,17 +583,7 @@
mRecorder.setVideoSource(MediaRecorder.VideoSource.CAMERA);
mRecorder.setAudioSource(MediaRecorder.AudioSource.DEFAULT);
- CamcorderProfile profile = null;
- for (int i: mPreferredProfiles) {
- if (CamcorderProfile.hasProfile(i)) {
- profile = CamcorderProfile.get(i);
- mRecorder.setProfile(profile);
- break;
- }
- }
-
- writeVideoMetaInfo(profile.videoFrameWidth, profile.videoFrameHeight,
- profile.videoFrameRate, fovW, fovH);
+ mRecorder.setProfile(mProfile);
try {
mRecorder.setOutputFile(getVideoRecFilePath());
@@ -689,8 +768,20 @@
*/
public void init() {
mSensorManager = (SensorManager)getSystemService(SENSOR_SERVICE);
+ if (mSensorManager == null) {
+ Log.e(TAG,"SensorManager is null!");
+ }
mRVSensor = mSensorManager.getDefaultSensor(Sensor.TYPE_ROTATION_VECTOR);
- mSensorManager.registerListener(this, mRVSensor, SENSOR_RATE);
+ if (mRVSensor != null) {
+ if (LOCAL_LOGV) Log.v(TAG, "Got RV Sensor");
+ }else {
+ Log.e(TAG, "Did not get RV sensor");
+ }
+ if(mSensorManager.registerListener(this, mRVSensor, SENSOR_RATE)) {
+ if (LOCAL_LOGV) Log.v(TAG,"Register listener successfull");
+ } else {
+ Log.e(TAG,"Register listener failed");
+ }
try {
mLogWriter= new OutputStreamWriter(
diff --git a/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVXCheckAnalyzer.java b/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVXCheckAnalyzer.java
index 9afd1a9..3dc7270 100644
--- a/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVXCheckAnalyzer.java
+++ b/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVXCheckAnalyzer.java
@@ -15,12 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.os.Debug;
import android.os.Environment;
+import android.os.PowerManager;
import android.util.JsonWriter;
import android.util.Log;
@@ -777,14 +777,16 @@
VideoMetaInfo meta = new VideoMetaInfo(new File(mPath, "videometa.json"));
int decimation = 1;
+ boolean use_timestamp = true;
+ // roughly determine if decimation is necessary
if (meta.fps > DECIMATION_FPS_TARGET) {
decimation = (int)(meta.fps / DECIMATION_FPS_TARGET);
meta.fps /=decimation;
}
VideoDecoderForOpenCV videoDecoder = new VideoDecoderForOpenCV(
- new File(mPath, "video.mp4"), decimation); // every 3 frame process 1 frame
+ new File(mPath, "video.mp4"), decimation);
Mat frame;
@@ -820,12 +822,17 @@
}
long startTime = System.nanoTime();
+ long [] ts = new long[1];
- while ((frame = videoDecoder.getFrame()) !=null) {
+ while ((frame = videoDecoder.getFrame(ts)) !=null) {
if (LOCAL_LOGV) {
Log.v(TAG, "got a frame " + i);
}
+ if (use_timestamp && ts[0] == -1) {
+ use_timestamp = false;
+ }
+
// has to be in front, as there are cases where execution
// will skip the later part of this while
i++;
@@ -873,8 +880,16 @@
// if error is reasonable, add it into the results
if (error < REPROJECTION_THREASHOLD) {
double [] rv = new double[3];
+ double timestamp;
+
rvec.get(0,0, rv);
- recs.add(new AttitudeRec((double) i / meta.fps, rodr2rpy(rv)));
+ if (use_timestamp) {
+ timestamp = (double)ts[0] / 1e6;
+ } else {
+ timestamp = (double) i / meta.fps;
+ }
+ if (LOCAL_LOGV) Log.v(TAG, String.format("Added frame %d ts = %f", i, timestamp));
+ recs.add(new AttitudeRec(timestamp, rodr2rpy(rv)));
}
if (OUTPUT_DEBUG_IMAGE) {
@@ -906,6 +921,8 @@
* One issue right now is that the glReadPixels is quite slow .. around 6.5ms for a 720p frame
*/
private class VideoDecoderForOpenCV implements Runnable {
+ static final String TAG = "VideoDecoderForOpenCV";
+
private MediaExtractor extractor=null;
private MediaCodec decoder=null;
private CtsMediaOutputSurface surface=null;
@@ -1031,7 +1048,7 @@
}
if (decoder == null) {
- Log.e("VideoDecoderForOpenCV", "Can't find video info!");
+ Log.e(TAG, "Can't find video info!");
return;
}
valid = true;
@@ -1060,6 +1077,7 @@
long timeoutUs = 10000;
int iframe = 0;
+ long frameTimestamp = 0;
while (!Thread.interrupted()) {
if (!isEOS) {
@@ -1076,8 +1094,12 @@
MediaCodec.BUFFER_FLAG_END_OF_STREAM);
isEOS = true;
} else {
- decoder.queueInputBuffer(inIndex, 0, sampleSize,
- extractor.getSampleTime(), 0);
+ frameTimestamp = extractor.getSampleTime();
+ decoder.queueInputBuffer(inIndex, 0, sampleSize, frameTimestamp, 0);
+ if (LOCAL_LOGD) {
+ Log.d(TAG, String.format("Frame %d sample time %f s",
+ iframe, (double)frameTimestamp/1e6));
+ }
extractor.advance();
}
}
@@ -1088,19 +1110,19 @@
switch (outIndex) {
case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
if (LOCAL_LOGD) {
- Log.d("VideoDecoderForOpenCV", "INFO_OUTPUT_BUFFERS_CHANGED");
+ Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED");
}
outputBuffers = decoder.getOutputBuffers();
break;
case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
outFormat = decoder.getOutputFormat();
if (LOCAL_LOGD) {
- Log.d("VideoDecoderForOpenCV", "New format " + outFormat);
+ Log.d(TAG, "New format " + outFormat);
}
break;
case MediaCodec.INFO_TRY_AGAIN_LATER:
if (LOCAL_LOGD) {
- Log.d("VideoDecoderForOpenCV", "dequeueOutputBuffer timed out!");
+ Log.d(TAG, "dequeueOutputBuffer timed out!");
}
break;
default:
@@ -1118,12 +1140,12 @@
if (doRender) {
surface.awaitNewImage();
surface.drawImage();
- if (LOCAL_LOGD) {
- Log.d("VideoDecoderForOpenCV", "Finish drawing a frame!");
+ if (LOCAL_LOGV) {
+ Log.v(TAG, "Finish drawing a frame!");
}
if ((iframe++ % mDecimation) == 0) {
//Send the frame for processing
- mMatBuffer.put();
+ mMatBuffer.put(frameTimestamp);
}
}
break;
@@ -1149,8 +1171,8 @@
* Get next valid frame
* @return Frame in OpenCV mat
*/
- public Mat getFrame() {
- return mMatBuffer.get();
+ public Mat getFrame(long ts[]) {
+ return mMatBuffer.get(ts);
}
/**
@@ -1168,6 +1190,7 @@
private Mat mat;
private byte[] bytes;
private ByteBuffer buf;
+ private long timestamp;
private boolean full;
private int mWidth, mHeight;
@@ -1180,6 +1203,7 @@
mat = new Mat(height, width, CvType.CV_8UC4); //RGBA
buf = ByteBuffer.allocateDirect(width*height*4);
bytes = new byte[width*height*4];
+ timestamp = -1;
mValid = true;
full = false;
@@ -1190,7 +1214,7 @@
notifyAll();
}
- public synchronized Mat get() {
+ public synchronized Mat get(long ts[]) {
if (!mValid) return null;
while (full == false) {
@@ -1204,9 +1228,11 @@
mat.put(0,0, bytes);
full = false;
notifyAll();
+ ts[0] = timestamp;
return mat;
}
- public synchronized void put() {
+
+ public synchronized void put(long ts) {
while (full) {
try {
wait();
@@ -1219,6 +1245,7 @@
buf.get(bytes);
buf.rewind();
+ timestamp = ts;
full = true;
notifyAll();
}
diff --git a/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVXCheckTestActivity.java b/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVXCheckTestActivity.java
index 9a74a0e..35c4d56 100644
--- a/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVXCheckTestActivity.java
+++ b/apps/CtsVerifier/src/com/android/cts/verifier/sensors/RVCVXCheckTestActivity.java
@@ -16,9 +16,10 @@
package com.android.cts.verifier.sensors;
-
+import android.content.Context;
import android.hardware.cts.helpers.SensorTestStateNotSupportedException;
import android.os.Bundle;
+import android.os.PowerManager;
import com.android.cts.verifier.sensors.base.SensorCtsVerifierTestActivity;
import com.android.cts.verifier.sensors.helpers.OpenCVLibrary;
@@ -82,7 +83,7 @@
while(retry-->0) {
try {
- Thread.sleep(100);
+ Thread.sleep(250);
} catch (InterruptedException e) {
//
}
@@ -146,7 +147,15 @@
// Analysis of recorded video and sensor data using RVCXAnalyzer
RVCVXCheckAnalyzer analyzer = new RVCVXCheckAnalyzer(mRecPath);
+
+ // acquire a partial wake lock just in case CPU fall asleep
+ PowerManager pm = (PowerManager) getSystemService(Context.POWER_SERVICE);
+ PowerManager.WakeLock wl = pm.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK,
+ "RVCVXCheckAnalyzer");
+
+ wl.acquire();
mReport = analyzer.processDataSet();
+ wl.release();
playSound();
vibrate(500);
