| /* |
| * Copyright (C) 2020 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package com.android.systemui.classifier; |
| |
| import static com.android.systemui.classifier.Classifier.BACK_GESTURE; |
| import static com.android.systemui.classifier.Classifier.GENERIC; |
| import static com.android.systemui.classifier.FalsingManagerProxy.FALSING_SUCCESS; |
| import static com.android.systemui.classifier.FalsingModule.BRIGHT_LINE_GESTURE_CLASSIFERS; |
| |
| import android.net.Uri; |
| import android.os.Build; |
| import android.util.IndentingPrintWriter; |
| import android.util.Log; |
| import android.view.accessibility.AccessibilityManager; |
| |
| import androidx.annotation.NonNull; |
| |
| import com.android.internal.logging.MetricsLogger; |
| import com.android.systemui.classifier.FalsingDataProvider.SessionListener; |
| import com.android.systemui.classifier.HistoryTracker.BeliefListener; |
| import com.android.systemui.dagger.qualifiers.TestHarness; |
| import com.android.systemui.plugins.FalsingManager; |
| import com.android.systemui.statusbar.policy.KeyguardStateController; |
| |
| import java.io.FileDescriptor; |
| import java.io.PrintWriter; |
| import java.util.ArrayDeque; |
| import java.util.ArrayList; |
| import java.util.Collection; |
| import java.util.Collections; |
| import java.util.List; |
| import java.util.Queue; |
| import java.util.Set; |
| import java.util.StringJoiner; |
| import java.util.stream.Collectors; |
| |
| import javax.inject.Inject; |
| import javax.inject.Named; |
| |
| /** |
| * FalsingManager designed to make clear why a touch was rejected. |
| */ |
| public class BrightLineFalsingManager implements FalsingManager { |
| |
| private static final String TAG = "FalsingManager"; |
| public static final boolean DEBUG = Log.isLoggable(TAG, Log.DEBUG); |
| |
| private static final int RECENT_INFO_LOG_SIZE = 40; |
| private static final int RECENT_SWIPE_LOG_SIZE = 20; |
| private static final double TAP_CONFIDENCE_THRESHOLD = 0.7; |
| private static final double FALSE_BELIEF_THRESHOLD = 0.9; |
| |
| private final FalsingDataProvider mDataProvider; |
| private final SingleTapClassifier mSingleTapClassifier; |
| private final DoubleTapClassifier mDoubleTapClassifier; |
| private final HistoryTracker mHistoryTracker; |
| private final KeyguardStateController mKeyguardStateController; |
| private AccessibilityManager mAccessibilityManager; |
| private final boolean mTestHarness; |
| private final MetricsLogger mMetricsLogger; |
| private int mIsFalseTouchCalls; |
| private static final Queue<String> RECENT_INFO_LOG = |
| new ArrayDeque<>(RECENT_INFO_LOG_SIZE + 1); |
| private static final Queue<DebugSwipeRecord> RECENT_SWIPES = |
| new ArrayDeque<>(RECENT_SWIPE_LOG_SIZE + 1); |
| |
| private final Collection<FalsingClassifier> mClassifiers; |
| private final List<FalsingBeliefListener> mFalsingBeliefListeners = new ArrayList<>(); |
| private List<FalsingTapListener> mFalsingTapListeners = new ArrayList<>(); |
| |
| private boolean mDestroyed; |
| |
| private final SessionListener mSessionListener = new SessionListener() { |
| @Override |
| public void onSessionEnded() { |
| mClassifiers.forEach(FalsingClassifier::onSessionEnded); |
| } |
| |
| @Override |
| public void onSessionStarted() { |
| mClassifiers.forEach(FalsingClassifier::onSessionStarted); |
| } |
| }; |
| |
| private final BeliefListener mBeliefListener = new BeliefListener() { |
| @Override |
| public void onBeliefChanged(double belief) { |
| logInfo(String.format( |
| "{belief=%s confidence=%s}", |
| mHistoryTracker.falseBelief(), |
| mHistoryTracker.falseConfidence())); |
| if (belief > FALSE_BELIEF_THRESHOLD) { |
| mFalsingBeliefListeners.forEach(FalsingBeliefListener::onFalse); |
| logInfo("Triggering False Event (Threshold: " + FALSE_BELIEF_THRESHOLD + ")"); |
| } |
| } |
| }; |
| |
| private final FalsingDataProvider.GestureFinalizedListener mGestureFinalizedListener = |
| new FalsingDataProvider.GestureFinalizedListener() { |
| @Override |
| public void onGestureFinalized(long completionTimeMs) { |
| if (mPriorResults != null) { |
| boolean boolResult = mPriorResults.stream().anyMatch( |
| FalsingClassifier.Result::isFalse); |
| |
| mPriorResults.forEach(result -> { |
| if (result.isFalse()) { |
| String reason = result.getReason(); |
| if (reason != null) { |
| logInfo(reason); |
| } |
| } |
| }); |
| |
| if (Build.IS_ENG || Build.IS_USERDEBUG) { |
| // Copy motion events, as the results returned by |
| // #getRecentMotionEvents are recycled elsewhere. |
| RECENT_SWIPES.add(new DebugSwipeRecord( |
| boolResult, |
| mPriorInteractionType, |
| mDataProvider.getRecentMotionEvents().stream().map( |
| motionEvent -> new XYDt( |
| (int) motionEvent.getX(), |
| (int) motionEvent.getY(), |
| (int) (motionEvent.getEventTime() |
| - motionEvent.getDownTime()))) |
| .collect(Collectors.toList()))); |
| while (RECENT_SWIPES.size() > RECENT_INFO_LOG_SIZE) { |
| RECENT_SWIPES.remove(); |
| } |
| } |
| |
| |
| mHistoryTracker.addResults(mPriorResults, completionTimeMs); |
| mPriorResults = null; |
| mPriorInteractionType = Classifier.GENERIC; |
| } else { |
| // Gestures that were not classified get treated as a false. |
| // Gestures that look like simple taps are less likely to be false |
| // than swipes. They may simply be mis-clicks. |
| double penalty = mSingleTapClassifier.isTap( |
| mDataProvider.getRecentMotionEvents(), 0).isFalse() |
| ? 0.7 : 0.8; |
| mHistoryTracker.addResults( |
| Collections.singleton( |
| FalsingClassifier.Result.falsed( |
| penalty, getClass().getSimpleName(), |
| "unclassified")), |
| completionTimeMs); |
| } |
| } |
| }; |
| |
| private Collection<FalsingClassifier.Result> mPriorResults; |
| private @Classifier.InteractionType int mPriorInteractionType = Classifier.GENERIC; |
| |
| @Inject |
| public BrightLineFalsingManager(FalsingDataProvider falsingDataProvider, |
| MetricsLogger metricsLogger, |
| @Named(BRIGHT_LINE_GESTURE_CLASSIFERS) Set<FalsingClassifier> classifiers, |
| SingleTapClassifier singleTapClassifier, DoubleTapClassifier doubleTapClassifier, |
| HistoryTracker historyTracker, KeyguardStateController keyguardStateController, |
| AccessibilityManager accessibilityManager, |
| @TestHarness boolean testHarness) { |
| mDataProvider = falsingDataProvider; |
| mMetricsLogger = metricsLogger; |
| mClassifiers = classifiers; |
| mSingleTapClassifier = singleTapClassifier; |
| mDoubleTapClassifier = doubleTapClassifier; |
| mHistoryTracker = historyTracker; |
| mKeyguardStateController = keyguardStateController; |
| mAccessibilityManager = accessibilityManager; |
| mTestHarness = testHarness; |
| |
| mDataProvider.addSessionListener(mSessionListener); |
| mDataProvider.addGestureCompleteListener(mGestureFinalizedListener); |
| mHistoryTracker.addBeliefListener(mBeliefListener); |
| } |
| |
| @Override |
| public boolean isClassifierEnabled() { |
| return true; |
| } |
| |
| @Override |
| public boolean isFalseTouch(@Classifier.InteractionType int interactionType) { |
| checkDestroyed(); |
| |
| mPriorInteractionType = interactionType; |
| if (skipFalsing(interactionType)) { |
| mPriorResults = getPassedResult(1); |
| logDebug("Skipped falsing"); |
| return false; |
| } |
| |
| final boolean[] localResult = {false}; |
| mPriorResults = mClassifiers.stream().map(falsingClassifier -> { |
| FalsingClassifier.Result r = falsingClassifier.classifyGesture( |
| interactionType, |
| mHistoryTracker.falseBelief(), |
| mHistoryTracker.falseConfidence()); |
| localResult[0] |= r.isFalse(); |
| |
| return r; |
| }).collect(Collectors.toList()); |
| |
| logDebug("False Gesture: " + localResult[0]); |
| |
| return localResult[0]; |
| } |
| |
| @Override |
| public boolean isSimpleTap() { |
| checkDestroyed(); |
| |
| FalsingClassifier.Result result = mSingleTapClassifier.isTap( |
| mDataProvider.getRecentMotionEvents(), 0); |
| mPriorResults = Collections.singleton(result); |
| |
| return !result.isFalse(); |
| } |
| |
| private void checkDestroyed() { |
| if (mDestroyed) { |
| Log.wtf(TAG, "Tried to use FalsingManager after being destroyed!"); |
| } |
| } |
| |
| @Override |
| public boolean isFalseTap(@Penalty int penalty) { |
| checkDestroyed(); |
| |
| if (skipFalsing(GENERIC)) { |
| mPriorResults = getPassedResult(1); |
| logDebug("Skipped falsing"); |
| return false; |
| } |
| |
| double falsePenalty = 0; |
| switch(penalty) { |
| case NO_PENALTY: |
| falsePenalty = 0; |
| break; |
| case LOW_PENALTY: |
| falsePenalty = 0.1; |
| break; |
| case MODERATE_PENALTY: |
| falsePenalty = 0.3; |
| break; |
| case HIGH_PENALTY: |
| falsePenalty = 0.6; |
| break; |
| } |
| |
| FalsingClassifier.Result singleTapResult = |
| mSingleTapClassifier.isTap(mDataProvider.getRecentMotionEvents().isEmpty() |
| ? mDataProvider.getPriorMotionEvents() |
| : mDataProvider.getRecentMotionEvents(), falsePenalty); |
| mPriorResults = Collections.singleton(singleTapResult); |
| |
| if (!singleTapResult.isFalse()) { |
| if (mDataProvider.isJustUnlockedWithFace()) { |
| // Immediately pass if a face is detected. |
| mPriorResults = getPassedResult(1); |
| logDebug("False Single Tap: false (face detected)"); |
| return false; |
| } else if (!isFalseDoubleTap()) { |
| // We must check double tapping before other heuristics. This is because |
| // the double tap will fail if there's only been one tap. We don't want that |
| // failure to be recorded in mPriorResults. |
| logDebug("False Single Tap: false (double tapped)"); |
| return false; |
| } else if (mHistoryTracker.falseBelief() > TAP_CONFIDENCE_THRESHOLD) { |
| mPriorResults = Collections.singleton( |
| FalsingClassifier.Result.falsed( |
| 0, getClass().getSimpleName(), "bad history")); |
| logDebug("False Single Tap: true (bad history)"); |
| mFalsingTapListeners.forEach(FalsingTapListener::onDoubleTapRequired); |
| return true; |
| } else { |
| mPriorResults = getPassedResult(0.1); |
| logDebug("False Single Tap: false (default)"); |
| return false; |
| } |
| |
| } else { |
| logDebug("False Single Tap: " + singleTapResult.isFalse() + " (simple)"); |
| return singleTapResult.isFalse(); |
| } |
| |
| } |
| |
| @Override |
| public boolean isFalseDoubleTap() { |
| checkDestroyed(); |
| |
| if (skipFalsing(GENERIC)) { |
| mPriorResults = getPassedResult(1); |
| logDebug("Skipped falsing"); |
| return false; |
| } |
| |
| FalsingClassifier.Result result = mDoubleTapClassifier.classifyGesture( |
| Classifier.GENERIC, |
| mHistoryTracker.falseBelief(), |
| mHistoryTracker.falseConfidence()); |
| mPriorResults = Collections.singleton(result); |
| logDebug("False Double Tap: " + result.isFalse()); |
| return result.isFalse(); |
| } |
| |
| private boolean skipFalsing(@Classifier.InteractionType int interactionType) { |
| return interactionType == BACK_GESTURE |
| || !mKeyguardStateController.isShowing() |
| || mTestHarness |
| || mDataProvider.isJustUnlockedWithFace() |
| || mDataProvider.isDocked() |
| || mAccessibilityManager.isEnabled(); |
| } |
| |
| @Override |
| public void onProximityEvent(ProximityEvent proximityEvent) { |
| // TODO: some of these classifiers might allow us to abort early, meaning we don't have to |
| // make these calls. |
| mClassifiers.forEach((classifier) -> classifier.onProximityEvent(proximityEvent)); |
| } |
| |
| @Override |
| public void onSuccessfulUnlock() { |
| if (mIsFalseTouchCalls != 0) { |
| mMetricsLogger.histogram(FALSING_SUCCESS, mIsFalseTouchCalls); |
| mIsFalseTouchCalls = 0; |
| } |
| } |
| |
| @Override |
| public boolean isUnlockingDisabled() { |
| return false; |
| } |
| |
| @Override |
| public boolean shouldEnforceBouncer() { |
| return false; |
| } |
| |
| @Override |
| public Uri reportRejectedTouch() { |
| return null; |
| } |
| |
| @Override |
| public boolean isReportingEnabled() { |
| return false; |
| } |
| |
| @Override |
| public void addFalsingBeliefListener(FalsingBeliefListener listener) { |
| mFalsingBeliefListeners.add(listener); |
| } |
| |
| @Override |
| public void removeFalsingBeliefListener(FalsingBeliefListener listener) { |
| mFalsingBeliefListeners.remove(listener); |
| } |
| |
| @Override |
| public void addTapListener(FalsingTapListener listener) { |
| mFalsingTapListeners.add(listener); |
| } |
| |
| @Override |
| public void removeTapListener(FalsingTapListener listener) { |
| mFalsingTapListeners.remove(listener); |
| } |
| |
| @Override |
| public void dump(@NonNull FileDescriptor fd, @NonNull PrintWriter pw, @NonNull String[] args) { |
| IndentingPrintWriter ipw = new IndentingPrintWriter(pw, " "); |
| ipw.println("BRIGHTLINE FALSING MANAGER"); |
| ipw.print("classifierEnabled="); |
| ipw.println(isClassifierEnabled() ? 1 : 0); |
| ipw.print("mJustUnlockedWithFace="); |
| ipw.println(mDataProvider.isJustUnlockedWithFace() ? 1 : 0); |
| ipw.print("isDocked="); |
| ipw.println(mDataProvider.isDocked() ? 1 : 0); |
| ipw.print("width="); |
| ipw.println(mDataProvider.getWidthPixels()); |
| ipw.print("height="); |
| ipw.println(mDataProvider.getHeightPixels()); |
| ipw.println(); |
| if (RECENT_SWIPES.size() != 0) { |
| ipw.println("Recent swipes:"); |
| ipw.increaseIndent(); |
| for (DebugSwipeRecord record : RECENT_SWIPES) { |
| ipw.println(record.getString()); |
| ipw.println(); |
| } |
| ipw.decreaseIndent(); |
| } else { |
| ipw.println("No recent swipes"); |
| } |
| ipw.println(); |
| ipw.println("Recent falsing info:"); |
| ipw.increaseIndent(); |
| for (String msg : RECENT_INFO_LOG) { |
| ipw.println(msg); |
| } |
| ipw.println(); |
| } |
| |
| @Override |
| public void cleanupInternal() { |
| mDestroyed = true; |
| mDataProvider.removeSessionListener(mSessionListener); |
| mDataProvider.removeGestureCompleteListener(mGestureFinalizedListener); |
| mClassifiers.forEach(FalsingClassifier::cleanup); |
| mFalsingBeliefListeners.clear(); |
| mHistoryTracker.removeBeliefListener(mBeliefListener); |
| } |
| |
| private static Collection<FalsingClassifier.Result> getPassedResult(double confidence) { |
| return Collections.singleton(FalsingClassifier.Result.passed(confidence)); |
| } |
| |
| static void logDebug(String msg) { |
| logDebug(msg, null); |
| } |
| |
| static void logDebug(String msg, Throwable throwable) { |
| if (DEBUG) { |
| Log.d(TAG, msg, throwable); |
| } |
| } |
| |
| static void logInfo(String msg) { |
| Log.i(TAG, msg); |
| RECENT_INFO_LOG.add(msg); |
| while (RECENT_INFO_LOG.size() > RECENT_INFO_LOG_SIZE) { |
| RECENT_INFO_LOG.remove(); |
| } |
| } |
| |
| static void logError(String msg) { |
| Log.e(TAG, msg); |
| } |
| |
| private static class DebugSwipeRecord { |
| private static final byte VERSION = 1; // opaque version number indicating format of data. |
| private final boolean mIsFalse; |
| private final int mInteractionType; |
| private final List<XYDt> mRecentMotionEvents; |
| |
| DebugSwipeRecord(boolean isFalse, int interactionType, |
| List<XYDt> recentMotionEvents) { |
| mIsFalse = isFalse; |
| mInteractionType = interactionType; |
| mRecentMotionEvents = recentMotionEvents; |
| } |
| |
| String getString() { |
| StringJoiner sj = new StringJoiner(","); |
| sj.add(Integer.toString(VERSION)) |
| .add(mIsFalse ? "1" : "0") |
| .add(Integer.toString(mInteractionType)); |
| for (XYDt event : mRecentMotionEvents) { |
| sj.add(event.toString()); |
| } |
| return sj.toString(); |
| } |
| } |
| |
| private static class XYDt { |
| private final int mX; |
| private final int mY; |
| private final int mDT; |
| |
| XYDt(int x, int y, int dT) { |
| mX = x; |
| mY = y; |
| mDT = dT; |
| } |
| |
| @Override |
| public String toString() { |
| return mX + "," + mY + "," + mDT; |
| } |
| } |
| } |