Add MagnificationGesturesObserver for A11y magnification
MagnificationGesturesObserver is used to detect the gestures when
users start to have interaction with the devices through fingers.
If users swipe, single tap or single tap and hold, it means users want
to interact with the UI. If users put two fingers down, it means users
are going to do gestures for magnification operation.
Test: MagnificationGesturesObserverTest
Bug: 149269335
Change-Id: I34ff0694b82ff76c462007f74938e79272d667b4
diff --git a/services/accessibility/java/com/android/server/accessibility/magnification/MagnificationGesturesObserver.java b/services/accessibility/java/com/android/server/accessibility/magnification/MagnificationGesturesObserver.java
new file mode 100644
index 0000000..a209086
--- /dev/null
+++ b/services/accessibility/java/com/android/server/accessibility/magnification/MagnificationGesturesObserver.java
@@ -0,0 +1,199 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.accessibility.magnification;
+
+import static com.android.server.accessibility.magnification.MagnificationGestureMatcher.GestureId;
+
+import android.annotation.MainThread;
+import android.annotation.NonNull;
+import android.annotation.Nullable;
+import android.annotation.SuppressLint;
+import android.util.Log;
+import android.util.Slog;
+import android.view.MotionEvent;
+
+import com.android.server.accessibility.gestures.GestureMatcher;
+
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Observes multiple {@link GestureMatcher} via {@link GesturesObserver}. In the observing duration,
+ * the event stream will be cached and sent through {@link Callback}.
+ *
+ */
+class MagnificationGesturesObserver implements GesturesObserver.Listener {
+
+ private static final String LOG_TAG = "MagnificationGesturesObserver";
+ @SuppressLint("LongLogTag")
+ private static final boolean DBG = Log.isLoggable(LOG_TAG, Log.DEBUG);
+
+ /**
+ * An Interface to determine if canceling detection and invoke the callbacks if the detection
+ * has a result.
+ */
+ interface Callback {
+ /**
+ * Called when receiving the event stream.
+ *
+ * @param motionEvent The received {@link MotionEvent}.
+ * @return {@code true} to cancel the detection.
+ */
+ boolean shouldStopDetection(MotionEvent motionEvent);
+
+ /**
+ * Called when the gesture is recognized.
+ *
+ * @param gestureId The gesture id of {@link GestureMatcher}.
+ * @param lastDownEventTime The time when receiving last {@link MotionEvent#ACTION_DOWN}.
+ * @param delayedEventQueue The collected event queue in whole detection duration.
+ * @param event The last event to determine the gesture. For the holding gestures, it's
+ * the last event before timeout.
+ *
+ * @see MagnificationGestureMatcher#GESTURE_SWIPE
+ * @see MagnificationGestureMatcher#GESTURE_TWO_FINGER_DOWN
+ */
+ void onGestureCompleted(@GestureId int gestureId, long lastDownEventTime,
+ List<MotionEventInfo> delayedEventQueue, MotionEvent event);
+
+ /**
+ * Called with the following conditions:
+ * <ol>
+ * <li> {@link #shouldStopDetection(MotionEvent)} returns {@code true}.
+ * <li> The system has decided an event stream doesn't match any known gesture.
+ * <ol>
+ *
+ * @param lastDownEventTime The time when receiving last {@link MotionEvent#ACTION_DOWN}.
+ * @param delayedEventQueue The collected event queue in whole detection duration.
+ * @param lastEvent The last event received before all matchers cancelling detection.
+ */
+ void onGestureCancelled(long lastDownEventTime,
+ List<MotionEventInfo> delayedEventQueue, MotionEvent lastEvent);
+ }
+
+ @Nullable private List<MotionEventInfo> mDelayedEventQueue;
+ private MotionEvent mLastEvent;
+ private long mLastDownEventTime = 0;
+ private final Callback mCallback;
+
+ private final GesturesObserver mGesturesObserver;
+
+ MagnificationGesturesObserver(@NonNull Callback callback, GestureMatcher... matchers) {
+ mGesturesObserver = new GesturesObserver(this, matchers);
+ mCallback = callback;
+ }
+
+ /**
+ * Processes a motion event and attempts to match it to one of the gestures.
+ *
+ * @param event the event as passed in from the event stream.
+ * @param rawEvent the original un-modified event. Useful for calculating movements in physical
+ * space.
+ * @param policyFlags the policy flags as passed in from the event stream.
+ * @return {@code true} if one of the gesture is matched.
+ */
+ @MainThread
+ boolean onMotionEvent(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
+ if (DBG) {
+ Slog.d(LOG_TAG, "DetectGesture: event = " + event);
+ }
+ cacheDelayedMotionEvent(event, rawEvent, policyFlags);
+ if (mCallback.shouldStopDetection(event)) {
+ notifyDetectionCancel();
+ return false;
+ }
+ if (event.getActionMasked() == MotionEvent.ACTION_DOWN) {
+ mLastDownEventTime = event.getDownTime();
+ }
+ return mGesturesObserver.onMotionEvent(event, rawEvent, policyFlags);
+ }
+
+ @Override
+ public void onGestureCompleted(int gestureId, MotionEvent event, MotionEvent rawEvent,
+ int policyFlags) {
+ if (DBG) {
+ Slog.d(LOG_TAG, "onGestureCompleted: " + MagnificationGestureMatcher.gestureIdToString(
+ gestureId) + " event = " + event);
+ }
+ final List<MotionEventInfo> delayEventQueue = mDelayedEventQueue;
+ mDelayedEventQueue = null;
+ mCallback.onGestureCompleted(gestureId, mLastDownEventTime, delayEventQueue,
+ event);
+ recycleLastEvent();
+ }
+
+ @Override
+ public void onGestureCancelled(MotionEvent event, MotionEvent rawEvent, int policyFlags) {
+ if (DBG) {
+ Slog.d(LOG_TAG, "onGestureCancelled: event = " + event);
+ }
+ notifyDetectionCancel();
+ }
+
+ private void notifyDetectionCancel() {
+ final List<MotionEventInfo> delayEventQueue = mDelayedEventQueue;
+ mDelayedEventQueue = null;
+ mCallback.onGestureCancelled(mLastDownEventTime, delayEventQueue,
+ mLastEvent);
+ recycleLastEvent();
+ }
+
+ /**
+ * Resets all state to default.
+ */
+ void clear() {
+ if (DBG) {
+ Slog.d(LOG_TAG, "clear:" + mDelayedEventQueue);
+ }
+ recycleLastEvent();
+ mLastDownEventTime = 0;
+ mGesturesObserver.clear();
+ if (mDelayedEventQueue != null) {
+ for (MotionEventInfo eventInfo2: mDelayedEventQueue) {
+ eventInfo2.recycle();
+ }
+ mDelayedEventQueue.clear();
+ mDelayedEventQueue = null;
+ }
+ }
+
+ private void recycleLastEvent() {
+ if (mLastEvent == null) {
+ return;
+ }
+ mLastEvent.recycle();
+ mLastEvent = null;
+ }
+
+ private void cacheDelayedMotionEvent(MotionEvent event, MotionEvent rawEvent,
+ int policyFlags) {
+ mLastEvent = MotionEvent.obtain(event);
+ MotionEventInfo info =
+ MotionEventInfo.obtain(event, rawEvent,
+ policyFlags);
+ if (mDelayedEventQueue == null) {
+ mDelayedEventQueue = new LinkedList<>();
+ }
+ mDelayedEventQueue.add(info);
+ }
+
+ @Override
+ public String toString() {
+ return "MagnificationGesturesObserver{"
+ + ", mDelayedEventQueue=" + mDelayedEventQueue + '}';
+ }
+}
diff --git a/services/accessibility/java/com/android/server/accessibility/magnification/MotionEventInfo.java b/services/accessibility/java/com/android/server/accessibility/magnification/MotionEventInfo.java
new file mode 100644
index 0000000..62a342f
--- /dev/null
+++ b/services/accessibility/java/com/android/server/accessibility/magnification/MotionEventInfo.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.accessibility.magnification;
+
+import android.annotation.Nullable;
+import android.view.MotionEvent;
+
+import com.android.server.accessibility.EventStreamTransformation;
+
+/**
+ * A data structure to store the parameters of
+ * {@link EventStreamTransformation#onMotionEvent(MotionEvent, MotionEvent, int)}.
+ */
+final class MotionEventInfo {
+
+ public MotionEvent mEvent;
+ public MotionEvent mRawEvent;
+ public int mPolicyFlags;
+
+ static MotionEventInfo obtain(MotionEvent event, MotionEvent rawEvent,
+ int policyFlags) {
+ return new MotionEventInfo(MotionEvent.obtain(event), MotionEvent.obtain(rawEvent),
+ policyFlags);
+ }
+
+ MotionEventInfo(MotionEvent event, MotionEvent rawEvent,
+ int policyFlags) {
+ mEvent = event;
+ mRawEvent = rawEvent;
+ mPolicyFlags = policyFlags;
+
+ }
+
+ void recycle() {
+ mEvent = recycleAndNullify(mEvent);
+ mRawEvent = recycleAndNullify(mRawEvent);
+ }
+
+ @Override
+ public String toString() {
+ return MotionEvent.actionToString(mEvent.getAction()).replace("ACTION_", "");
+ }
+
+ private static MotionEvent recycleAndNullify(@Nullable MotionEvent event) {
+ if (event != null) {
+ event.recycle();
+ }
+ return null;
+ }
+}
diff --git a/services/tests/servicestests/src/com/android/server/accessibility/magnification/MagnificationGesturesObserverTest.java b/services/tests/servicestests/src/com/android/server/accessibility/magnification/MagnificationGesturesObserverTest.java
new file mode 100644
index 0000000..895fb17
--- /dev/null
+++ b/services/tests/servicestests/src/com/android/server/accessibility/magnification/MagnificationGesturesObserverTest.java
@@ -0,0 +1,167 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.server.accessibility.magnification;
+
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import android.app.Instrumentation;
+import android.content.Context;
+import android.view.Display;
+import android.view.MotionEvent;
+import android.view.ViewConfiguration;
+
+import androidx.test.InstrumentationRegistry;
+
+import com.android.server.accessibility.utils.TouchEventGenerator;
+
+import junit.framework.Assert;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.ArgumentMatcher;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+import java.util.List;
+
+/**
+ * Tests for MagnificationGesturesObserver.
+ */
+public class MagnificationGesturesObserverTest {
+
+ private static final float DEFAULT_X = 100f;
+ private static final float DEFAULT_Y = 100f;
+
+ @Mock
+ private MagnificationGesturesObserver.Callback mCallback;
+ @Captor
+ private ArgumentCaptor<List<MotionEventInfo>> mEventInfoArgumentCaptor;
+
+ private Context mContext;
+ private Instrumentation mInstrumentation;
+ private MagnificationGesturesObserver mObserver;
+
+ @Before
+ public void setUp() {
+ MockitoAnnotations.initMocks(this);
+ mContext = InstrumentationRegistry.getContext();
+ mInstrumentation = InstrumentationRegistry.getInstrumentation();
+ mObserver = new MagnificationGesturesObserver(mCallback, new SimpleSwipe(mContext),
+ new TwoFingersDown(mContext));
+ }
+
+ @Test
+ public void onActionMove_onGestureCanceled() {
+ final MotionEvent moveEvent = TouchEventGenerator.moveEvent(Display.DEFAULT_DISPLAY,
+ DEFAULT_X , DEFAULT_Y);
+
+ mInstrumentation.runOnMainSync(() -> {
+ mObserver.onMotionEvent(moveEvent, moveEvent, 0);
+ });
+
+ verify(mCallback).onGestureCancelled(eq(0L),
+ mEventInfoArgumentCaptor.capture(), argThat(new MotionEventMatcher(moveEvent)));
+ verifyCacheMotionEvents(mEventInfoArgumentCaptor.getValue(), moveEvent);
+ }
+
+ @Test
+ public void onActionDown_shouldNotDetection_onGestureCanceled() {
+ when(mCallback.shouldStopDetection(any(MotionEvent.class))).thenReturn(true);
+ final MotionEvent downEvent = TouchEventGenerator.downEvent(Display.DEFAULT_DISPLAY,
+ DEFAULT_X , DEFAULT_Y);
+
+ mInstrumentation.runOnMainSync(() -> {
+ mObserver.onMotionEvent(downEvent, downEvent, 0);
+ });
+
+ verify(mCallback).onGestureCancelled(eq(0L),
+ mEventInfoArgumentCaptor.capture(), argThat(new MotionEventMatcher(downEvent)));
+ verifyCacheMotionEvents(mEventInfoArgumentCaptor.getValue(), downEvent);
+ }
+
+ @Test
+ public void onMotionEvent_unrecognizedEvents_onDetectionCanceledAfterTimeout() {
+ final MotionEvent downEvent = TouchEventGenerator.downEvent(Display.DEFAULT_DISPLAY,
+ DEFAULT_X, DEFAULT_Y);
+ final int timeoutMillis = MagnificationGestureMatcher.getMagnificationMultiTapTimeout(
+ mContext) + 100;
+
+ mInstrumentation.runOnMainSync(() -> {
+ mObserver.onMotionEvent(downEvent, downEvent, 0);
+ });
+
+ verify(mCallback, timeout(timeoutMillis)).onGestureCancelled(eq(downEvent.getDownTime()),
+ mEventInfoArgumentCaptor.capture(), argThat(new MotionEventMatcher(downEvent)));
+ verifyCacheMotionEvents(mEventInfoArgumentCaptor.getValue(), downEvent);
+ }
+
+ @Test
+ public void sendEventsOfSwiping_onGestureCompleted() {
+ final MotionEvent downEvent = TouchEventGenerator.downEvent(Display.DEFAULT_DISPLAY,
+ DEFAULT_X, DEFAULT_Y);
+ final float swipeDistance = ViewConfiguration.get(mContext).getScaledTouchSlop();
+ final MotionEvent moveEvent = TouchEventGenerator.moveEvent(Display.DEFAULT_DISPLAY,
+ DEFAULT_X + swipeDistance, DEFAULT_Y + swipeDistance);
+
+ mInstrumentation.runOnMainSync(() -> {
+ mObserver.onMotionEvent(downEvent, downEvent, 0);
+ mObserver.onMotionEvent(moveEvent, moveEvent, 0);
+ });
+
+ verify(mCallback).onGestureCompleted(eq(MagnificationGestureMatcher.GESTURE_SWIPE),
+ eq(downEvent.getDownTime()), mEventInfoArgumentCaptor.capture(),
+ argThat(new MotionEventMatcher(moveEvent)));
+ verifyCacheMotionEvents(mEventInfoArgumentCaptor.getValue(), downEvent, moveEvent);
+ }
+
+ private static class MotionEventMatcher implements ArgumentMatcher<MotionEvent> {
+
+ private final MotionEvent mExpectedEvent;
+ MotionEventMatcher(MotionEvent motionEvent) {
+ mExpectedEvent = motionEvent;
+ }
+
+ @Override
+ public boolean matches(MotionEvent actualEvent) {
+ return compareMotionEvent(mExpectedEvent, actualEvent);
+ }
+ }
+
+ private static boolean compareMotionEvent(MotionEvent expectedEvent, MotionEvent actualEvent) {
+ if (expectedEvent == null || actualEvent == null) {
+ return false;
+ }
+ return expectedEvent.toString().contentEquals(actualEvent.toString());
+ }
+
+ private static void verifyCacheMotionEvents(List<MotionEventInfo> actualEvents,
+ MotionEvent... expectedEvents) {
+ Assert.assertEquals("events size doesn't match", expectedEvents.length,
+ actualEvents.size());
+ for (int i = 0; i < actualEvents.size(); i++) {
+ Assert.assertTrue(compareMotionEvent(expectedEvents[i], actualEvents.get(i).mEvent));
+ }
+ }
+}
