Improve fairness of Sensor Timestamp Jitter Test
Changes:
1. The threshold is calculated based on the "asked" sensor rate to
avoid failures that caused by actual sensor rate being bumpped up.
2. Estimation of the true center is no longer necessary. The range of
timestamp interval is tested against 2x of the allowed single-sided
variance.
3. The top and bottom 2.5% of timestamp interval samples are excluded
from evaluation. This conform to previously used 95 percentile
rule, yet is slightly different in terms of implementation.
Related bug:
* b/23430923
* b/23948014
Change-Id: I1f444514c2df63129d9e0ce404afe3ae7ae46eb0
diff --git a/tests/tests/hardware/src/android/hardware/cts/helpers/SensorCtsHelper.java b/tests/tests/hardware/src/android/hardware/cts/helpers/SensorCtsHelper.java
index 55465ac..a69dc54 100644
--- a/tests/tests/hardware/src/android/hardware/cts/helpers/SensorCtsHelper.java
+++ b/tests/tests/hardware/src/android/hardware/cts/helpers/SensorCtsHelper.java
@@ -38,21 +38,27 @@
private SensorCtsHelper() {}
/**
- * Get the value of the 95th percentile using nearest rank algorithm.
+ * Get percentiles using nearest rank algorithm.
*
- * @throws IllegalArgumentException if the collection is null or empty
+ * @throws IllegalArgumentException if the collection or percentiles is null or empty
*/
- public static <TValue extends Comparable<? super TValue>> TValue get95PercentileValue(
- Collection<TValue> collection) {
+ public static <TValue extends Comparable<? super TValue>> List<TValue> getPercentileValue(
+ Collection<TValue> collection, float[] percentiles) {
validateCollection(collection);
+ if(percentiles == null || percentiles.length == 0) {
+ throw new IllegalStateException("percentiles cannot be null or empty");
+ }
List<TValue> arrayCopy = new ArrayList<TValue>(collection);
Collections.sort(arrayCopy);
- // zero-based array index
- int arrayIndex = (int) Math.round(arrayCopy.size() * 0.95 + .5) - 1;
-
- return arrayCopy.get(arrayIndex);
+ List<TValue> percentileValues = new ArrayList<TValue>();
+ for (float p : percentiles) {
+ // zero-based array index
+ int arrayIndex = (int) Math.round(arrayCopy.size() * p + .5) - 1;
+ percentileValues.add(arrayCopy.get(arrayIndex));
+ }
+ return percentileValues;
}
/**
diff --git a/tests/tests/hardware/src/android/hardware/cts/helpers/sensorverification/JitterVerification.java b/tests/tests/hardware/src/android/hardware/cts/helpers/sensorverification/JitterVerification.java
index d246ec5..d9ee5b8 100644
--- a/tests/tests/hardware/src/android/hardware/cts/helpers/sensorverification/JitterVerification.java
+++ b/tests/tests/hardware/src/android/hardware/cts/helpers/sensorverification/JitterVerification.java
@@ -43,15 +43,22 @@
// sensorType: threshold (% of expected period)
private static final SparseIntArray DEFAULTS = new SparseIntArray(12);
- // Max allowed jitter (in percentage).
+ // Max allowed jitter in +/- sense (in percentage).
private static final int GRACE_FACTOR = 2;
private static final int THRESHOLD_PERCENT_FOR_HIFI_SENSORS = 1 * GRACE_FACTOR;
+
+ // Margin sample intervals that considered outliers, lower and higher margin is discarded
+ // before verification
+ private static final float OUTLIER_MARGIN = 0.025f; //2.5%
+
static {
// Use a method so that the @deprecation warning can be set for that method only
setDefaults();
}
- private final int mThresholdAsPercentage;
+ private final float mOutlierMargin;
+ private final long mThresholdNs;
+ private final long mExpectedPeriodNs; // for error message only
private final List<Long> mTimestamps = new LinkedList<Long>();
/**
@@ -59,8 +66,10 @@
*
* @param thresholdAsPercentage the acceptable margin of error as a percentage
*/
- public JitterVerification(int thresholdAsPercentage) {
- mThresholdAsPercentage = thresholdAsPercentage;
+ public JitterVerification(float outlierMargin, long thresholdNs, long expectedPeriodNs) {
+ mExpectedPeriodNs = expectedPeriodNs;
+ mOutlierMargin = outlierMargin;
+ mThresholdNs = thresholdNs;
}
/**
@@ -71,16 +80,20 @@
*/
public static JitterVerification getDefault(TestSensorEnvironment environment) {
int sensorType = environment.getSensor().getType();
- int threshold = DEFAULTS.get(sensorType, -1);
- if (threshold == -1) {
+
+ int thresholdPercent = DEFAULTS.get(sensorType, -1);
+ if (thresholdPercent == -1) {
return null;
}
boolean hasHifiSensors = environment.getContext().getPackageManager().hasSystemFeature(
PackageManager.FEATURE_HIFI_SENSORS);
if (hasHifiSensors) {
- threshold = THRESHOLD_PERCENT_FOR_HIFI_SENSORS;
+ thresholdPercent = THRESHOLD_PERCENT_FOR_HIFI_SENSORS;
}
- return new JitterVerification(threshold);
+
+ long expectedPeriodNs = (long) environment.getExpectedSamplingPeriodUs() * 1000;
+ long jitterThresholdNs = expectedPeriodNs * thresholdPercent * 2 / 100; // *2 is for +/-
+ return new JitterVerification(OUTLIER_MARGIN, jitterThresholdNs, expectedPeriodNs);
}
/**
@@ -99,24 +112,33 @@
return;
}
- List<Double> jitters = getJitterValues();
- double jitter95PercentileNs = SensorCtsHelper.get95PercentileValue(jitters);
- long firstTimestamp = mTimestamps.get(0);
- long lastTimestamp = mTimestamps.get(timestampsCount - 1);
- long measuredPeriodNs = (lastTimestamp - firstTimestamp) / (timestampsCount - 1);
- double jitter95PercentilePercent = (jitter95PercentileNs * 100.0) / measuredPeriodNs;
- stats.addValue(SensorStats.JITTER_95_PERCENTILE_PERCENT_KEY, jitter95PercentilePercent);
+ List<Long> deltas = getDeltaValues();
+ float percentiles[] = new float[2];
+ percentiles[0] = mOutlierMargin;
+ percentiles[1] = 1 - percentiles[0];
- boolean success = (jitter95PercentilePercent < mThresholdAsPercentage);
+ List<Long> percentileValues = SensorCtsHelper.getPercentileValue(deltas, percentiles);
+ double normalizedRange =
+ (double)(percentileValues.get(1) - percentileValues.get(0)) / mThresholdNs;
+
+ double percentageJitter =
+ (double)(percentileValues.get(1) - percentileValues.get(0)) /
+ mExpectedPeriodNs / 2 * 100; //one side variation comparing to sample time
+
+ stats.addValue(SensorStats.JITTER_95_PERCENTILE_PERCENT_KEY, percentageJitter);
+
+ boolean success = normalizedRange <= 1.0;
stats.addValue(PASSED_KEY, success);
if (!success) {
String message = String.format(
- "Jitter out of range: measured period=%dns, jitter(95th percentile)=%.2f%%"
- + " (expected < %d%%)",
- measuredPeriodNs,
- jitter95PercentilePercent,
- mThresholdAsPercentage);
+ "Jitter out of range: requested period = %dns, " +
+ "jitter min, max, range (95th percentile) = (%dns, %dns, %dns), " +
+ "jitter expected range <= %dns",
+ mExpectedPeriodNs,
+ percentileValues.get(0), percentileValues.get(1),
+ percentileValues.get(1) - percentileValues.get(0),
+ mThresholdNs);
Assert.fail(message);
}
}
@@ -126,7 +148,7 @@
*/
@Override
public JitterVerification clone() {
- return new JitterVerification(mThresholdAsPercentage);
+ return new JitterVerification(mOutlierMargin, mThresholdNs, mExpectedPeriodNs);
}
/**
@@ -138,19 +160,14 @@
}
/**
- * Get the list of all jitter values. Exposed for unit testing.
+ * Get the list of delta values. Exposed for unit testing.
*/
- List<Double> getJitterValues() {
+ List<Long> getDeltaValues() {
List<Long> deltas = new ArrayList<Long>(mTimestamps.size() - 1);
for (int i = 1; i < mTimestamps.size(); i++) {
deltas.add(mTimestamps.get(i) - mTimestamps.get(i - 1));
}
- double deltaMean = StatisticsUtils.getMean(deltas);
- List<Double> jitters = new ArrayList<Double>(deltas.size());
- for (long delta : deltas) {
- jitters.add(Math.abs(delta - deltaMean));
- }
- return jitters;
+ return deltas;
}
@SuppressWarnings("deprecation")
diff --git a/tests/tests/hardware/src/android/hardware/cts/helpers/sensorverification/JitterVerificationTest.java b/tests/tests/hardware/src/android/hardware/cts/helpers/sensorverification/JitterVerificationTest.java
index 50e288c..79c3b65 100644
--- a/tests/tests/hardware/src/android/hardware/cts/helpers/sensorverification/JitterVerificationTest.java
+++ b/tests/tests/hardware/src/android/hardware/cts/helpers/sensorverification/JitterVerificationTest.java
@@ -27,11 +27,10 @@
import java.util.List;
/**
- * Tests for {@link EventOrderingVerification}.
+ * Tests for {@link JitterVerification}.
*/
public class JitterVerificationTest extends TestCase {
-
public void testVerify() {
final int SAMPLE_SIZE = 100;
// for unit testing the verification, only the parameter 'sensorMightHaveMoreListeners' is
@@ -67,37 +66,36 @@
} catch (AssertionError e) {
// Expected;
}
- verifyStats(stats, false, 47.34);
+ verifyStats(stats, false, 50); // 500 us range divide by 1ms requested sample time x 100%
}
- public void testCalculateJitter() {
+ public void testCalculateDelta() {
long[] timestamps = new long[]{0, 1, 2, 3, 4};
JitterVerification verification = getVerification(1, timestamps);
- List<Double> jitterValues = verification.getJitterValues();
- assertEquals(4, jitterValues.size());
- assertEquals(0.0, jitterValues.get(0));
- assertEquals(0.0, jitterValues.get(1));
- assertEquals(0.0, jitterValues.get(2));
- assertEquals(0.0, jitterValues.get(3));
+ List<Long> deltaValues = verification.getDeltaValues();
+ assertEquals(4, deltaValues.size());
+ assertEquals(1, deltaValues.get(0).doubleValue());
+ assertEquals(1, deltaValues.get(1).doubleValue());
+ assertEquals(1, deltaValues.get(2).doubleValue());
+ assertEquals(1, deltaValues.get(3).doubleValue());
timestamps = new long[]{0, 0, 2, 4, 4};
verification = getVerification(1, timestamps);
- jitterValues = verification.getJitterValues();
- assertEquals(4, jitterValues.size());
- assertEquals(1.0, jitterValues.get(0));
- assertEquals(1.0, jitterValues.get(1));
- assertEquals(1.0, jitterValues.get(2));
- assertEquals(1.0, jitterValues.get(3));
+ deltaValues = verification.getDeltaValues();
+ assertEquals(4, deltaValues.size());
+ assertEquals(0, deltaValues.get(0).doubleValue());
+ assertEquals(2, deltaValues.get(1).doubleValue());
+ assertEquals(2, deltaValues.get(2).doubleValue());
+ assertEquals(0, deltaValues.get(3).doubleValue());
timestamps = new long[]{0, 1, 4, 9, 16};
verification = getVerification(1, timestamps);
- jitterValues = verification.getJitterValues();
- assertEquals(4, jitterValues.size());
- assertEquals(4, jitterValues.size());
- assertEquals(3.0, jitterValues.get(0));
- assertEquals(1.0, jitterValues.get(1));
- assertEquals(1.0, jitterValues.get(2));
- assertEquals(3.0, jitterValues.get(3));
+ deltaValues = verification.getDeltaValues();
+ assertEquals(4, deltaValues.size());
+ assertEquals(1, deltaValues.get(0).doubleValue());
+ assertEquals(3, deltaValues.get(1).doubleValue());
+ assertEquals(5, deltaValues.get(2).doubleValue());
+ assertEquals(7, deltaValues.get(3).doubleValue());
}
private static JitterVerification getVerification(int threshold, long ... timestamps) {
@@ -105,16 +103,20 @@
for (long timestamp : timestamps) {
events.add(new TestSensorEvent(null, timestamp, 0, null));
}
- JitterVerification verification = new JitterVerification(threshold);
+ long samplePeriodNs = 1000*1000; //1000Hz
+ long jitterThresholdNs = 20*1000; // 2%
+
+ JitterVerification verification =
+ new JitterVerification(threshold, jitterThresholdNs, samplePeriodNs);
verification.addSensorEvents(events);
return verification;
}
- private void verifyStats(SensorStats stats, boolean passed, double jitter95) {
+ private void verifyStats(SensorStats stats, boolean passed, double normalizedRange) {
assertEquals(passed, stats.getValue(JitterVerification.PASSED_KEY));
assertEquals(
- jitter95,
+ normalizedRange,
(Double) stats.getValue(SensorStats.JITTER_95_PERCENTILE_PERCENT_KEY),
- 0.1);
+ 0.01);
}
}
