| /* |
| * Copyright (C) 2021 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package android.accessibilityservice; |
| |
| import android.annotation.IntDef; |
| import android.annotation.NonNull; |
| import android.annotation.Nullable; |
| import android.os.RemoteException; |
| import android.util.ArrayMap; |
| import android.view.MotionEvent; |
| import android.view.accessibility.AccessibilityInteractionClient; |
| |
| import java.lang.annotation.Retention; |
| import java.lang.annotation.RetentionPolicy; |
| import java.util.LinkedList; |
| import java.util.Queue; |
| import java.util.concurrent.Executor; |
| |
| /** |
| * This class allows a service to handle touch exploration and the detection of specialized |
| * accessibility gestures. The service receives motion events and can match those motion events |
| * against the gestures it supports. The service can also request the framework enter three other |
| * states of operation for the duration of this interaction. Upon entering any of these states the |
| * framework will take over and the service will not receive motion events until the start of a new |
| * interaction. The states are as follows: |
| * |
| * <ul> |
| * <li>The service can tell the framework that this interaction is touch exploration. The user is |
| * trying to explore the screen rather than manipulate it. The framework will then convert the |
| * motion events to hover events to support touch exploration. |
| * <li>The service can tell the framework that this interaction is a dragging interaction where |
| * two fingers are used to execute a one-finger gesture such as scrolling the screen. The |
| * service must specify which of the two fingers should be passed through to rest of the input |
| * pipeline. |
| * <li>Finally, the service can request that the framework delegate this interaction, meaning pass |
| * it through to the rest of the input pipeline as-is. |
| * </ul> |
| * |
| * When {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE } is enabled, this |
| * controller will receive all motion events received by the framework for the specified display |
| * when not touch-exploring or delegating. If the service classifies this interaction as touch |
| * exploration or delegating the framework will stop sending motion events to the service for the |
| * duration of this interaction. If the service classifies this interaction as a dragging |
| * interaction the framework will send motion events to the service to allow the service to |
| * determine if the interaction still qualifies as dragging or if it has become a delegating |
| * interaction. If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE } is disabled |
| * this controller will not receive any motion events because touch interactions are being passed |
| * through to the input pipeline unaltered. |
| * Note that {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE } |
| * requires setting {@link android.R.attr#canRequestTouchExplorationMode} as well. |
| */ |
| public final class TouchInteractionController { |
| /** The state where the user is not touching the screen. */ |
| public static final int STATE_CLEAR = 0; |
| /** |
| * The state where the user is touching the screen and the service is receiving motion events. |
| */ |
| public static final int STATE_TOUCH_INTERACTING = 1; |
| /** |
| * The state where the user is explicitly exploring the screen. The service is not receiving |
| * motion events. |
| */ |
| public static final int STATE_TOUCH_EXPLORING = 2; |
| /** |
| * The state where the user is dragging with two fingers. The service is not receiving motion |
| * events. The selected finger is being dispatched to the rest of the input pipeline to execute |
| * the drag. |
| */ |
| public static final int STATE_DRAGGING = 3; |
| /** |
| * The user is performing a gesture which is being passed through to the input pipeline as-is. |
| * The service is not receiving motion events. |
| */ |
| public static final int STATE_DELEGATING = 4; |
| |
| @IntDef({ |
| STATE_CLEAR, |
| STATE_TOUCH_INTERACTING, |
| STATE_TOUCH_EXPLORING, |
| STATE_DRAGGING, |
| STATE_DELEGATING |
| }) |
| @Retention(RetentionPolicy.SOURCE) |
| private @interface State {} |
| |
| // The maximum number of pointers that can be touching the screen at once. (See MAX_POINTER_ID |
| // in frameworks/native/include/input/Input.h) |
| private static final int MAX_POINTER_COUNT = 32; |
| |
| private final AccessibilityService mService; |
| private final Object mLock; |
| private final int mDisplayId; |
| private boolean mServiceDetectsGestures; |
| /** Map of callbacks to executors. Lazily created when adding the first callback. */ |
| private ArrayMap<Callback, Executor> mCallbacks; |
| // A list of motion events that should be queued until a pending transition has taken place. |
| private Queue<MotionEvent> mQueuedMotionEvents = new LinkedList<>(); |
| // Whether this controller is waiting for a state transition. |
| // Motion events will be queued and sent to listeners after the transition has taken place. |
| private boolean mStateChangeRequested = false; |
| |
| // The current state of the display. |
| private int mState = STATE_CLEAR; |
| |
| TouchInteractionController( |
| @NonNull AccessibilityService service, @NonNull Object lock, int displayId) { |
| mDisplayId = displayId; |
| mLock = lock; |
| mService = service; |
| } |
| |
| /** |
| * Adds the specified callback to the list of callbacks. The callback will |
| * run using on the specified {@link Executor}', or on the service's main thread if the |
| * Executor is {@code null}. |
| * @param callback the callback to add, must be non-null |
| * @param executor the executor for this callback, or {@code null} to execute on the service's |
| * main thread |
| */ |
| public void registerCallback(@Nullable Executor executor, @NonNull Callback callback) { |
| synchronized (mLock) { |
| if (mCallbacks == null) { |
| mCallbacks = new ArrayMap<>(); |
| } |
| mCallbacks.put(callback, executor); |
| if (mCallbacks.size() == 1) { |
| setServiceDetectsGestures(true); |
| } |
| } |
| } |
| |
| /** |
| * Unregisters the specified callback. |
| * |
| * @param callback the callback to remove, must be non-null |
| * @return {@code true} if the callback was removed, {@code false} otherwise |
| */ |
| public boolean unregisterCallback(@NonNull Callback callback) { |
| if (mCallbacks == null) { |
| return false; |
| } |
| synchronized (mLock) { |
| boolean result = mCallbacks.remove(callback) != null; |
| if (result && mCallbacks.size() == 0) { |
| setServiceDetectsGestures(false); |
| } |
| return result; |
| } |
| } |
| |
| /** |
| * Removes all callbacks and returns control of touch interactions to the framework. |
| */ |
| public void unregisterAllCallbacks() { |
| if (mCallbacks != null) { |
| synchronized (mLock) { |
| mCallbacks.clear(); |
| setServiceDetectsGestures(false); |
| } |
| } |
| } |
| |
| /** |
| * Dispatches motion events to any registered callbacks. This should be called on the service's |
| * main thread. |
| */ |
| void onMotionEvent(MotionEvent event) { |
| if (mStateChangeRequested) { |
| mQueuedMotionEvents.add(event); |
| } else { |
| sendEventToAllListeners(event); |
| } |
| } |
| |
| private void sendEventToAllListeners(MotionEvent event) { |
| final ArrayMap<Callback, Executor> entries; |
| synchronized (mLock) { |
| // callbacks may remove themselves. Perform a shallow copy to avoid concurrent |
| // modification. |
| entries = new ArrayMap<>(mCallbacks); |
| } |
| for (int i = 0, count = entries.size(); i < count; i++) { |
| final Callback callback = entries.keyAt(i); |
| final Executor executor = entries.valueAt(i); |
| if (executor != null) { |
| executor.execute(() -> callback.onMotionEvent(event)); |
| } else { |
| // We're already on the main thread, just run the callback. |
| callback.onMotionEvent(event); |
| } |
| } |
| } |
| |
| /** |
| * Dispatches motion events to any registered callbacks. This should be called on the service's |
| * main thread. |
| */ |
| void onStateChanged(@State int state) { |
| mState = state; |
| final ArrayMap<Callback, Executor> entries; |
| synchronized (mLock) { |
| // callbacks may remove themselves. Perform a shallow copy to avoid concurrent |
| // modification. |
| entries = new ArrayMap<>(mCallbacks); |
| } |
| for (int i = 0, count = entries.size(); i < count; i++) { |
| final Callback callback = entries.keyAt(i); |
| final Executor executor = entries.valueAt(i); |
| if (executor != null) { |
| executor.execute(() -> callback.onStateChanged(state)); |
| } else { |
| // We're already on the main thread, just run the callback. |
| callback.onStateChanged(state); |
| } |
| } |
| mStateChangeRequested = false; |
| while (mQueuedMotionEvents.size() > 0) { |
| sendEventToAllListeners(mQueuedMotionEvents.poll()); |
| } |
| } |
| |
| /** |
| * When {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled, this |
| * controller will receive all motion events received by the framework for the specified display |
| * when not touch-exploring, delegating, or dragging. This allows the service to detect its own |
| * gestures, and use its own logic to judge when the framework should start touch-exploring, |
| * delegating, or dragging. If {@link |
| * AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE } is disabled this flag has no |
| * effect. |
| * |
| * @see AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE |
| */ |
| private void setServiceDetectsGestures(boolean mode) { |
| final IAccessibilityServiceConnection connection = |
| AccessibilityInteractionClient.getInstance() |
| .getConnection(mService.getConnectionId()); |
| if (connection != null) { |
| try { |
| connection.setServiceDetectsGesturesEnabled(mDisplayId, mode); |
| mServiceDetectsGestures = mode; |
| } catch (RemoteException re) { |
| throw new RuntimeException(re); |
| } |
| } |
| } |
| |
| /** |
| * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at |
| * least one callback has been added for this display this function tells the framework to |
| * initiate touch exploration. Touch exploration will continue for the duration of this |
| * interaction. |
| */ |
| public void requestTouchExploration() { |
| validateTransitionRequest(); |
| final IAccessibilityServiceConnection connection = |
| AccessibilityInteractionClient.getInstance() |
| .getConnection(mService.getConnectionId()); |
| if (connection != null) { |
| try { |
| connection.requestTouchExploration(mDisplayId); |
| } catch (RemoteException re) { |
| throw new RuntimeException(re); |
| } |
| mStateChangeRequested = true; |
| } |
| } |
| |
| /** |
| * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If |
| * {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least |
| * one callback has been added, this function tells the framework to initiate a dragging |
| * interaction using the specified pointer. The pointer's movements will be passed through to |
| * the rest of the input pipeline. Dragging is often used to perform two-finger scrolling. |
| * |
| * @param pointerId the pointer to be passed through to the rest of the input pipeline. If the |
| * pointer id is valid but not actually present on the screen it will be ignored. |
| * @throws IllegalArgumentException if the pointer id is outside of the allowed range. |
| */ |
| public void requestDragging(int pointerId) { |
| validateTransitionRequest(); |
| if (pointerId < 0 || pointerId > MAX_POINTER_COUNT) { |
| throw new IllegalArgumentException("Invalid pointer id: " + pointerId); |
| } |
| final IAccessibilityServiceConnection connection = |
| AccessibilityInteractionClient.getInstance() |
| .getConnection(mService.getConnectionId()); |
| if (connection != null) { |
| try { |
| connection.requestDragging(mDisplayId, pointerId); |
| } catch (RemoteException re) { |
| throw new RuntimeException(re); |
| } |
| mStateChangeRequested = true; |
| } |
| } |
| |
| /** |
| * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If |
| * {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least |
| * one callback has been added, this function tells the framework to initiate a delegating |
| * interaction. Motion events will be passed through as-is to the rest of the input pipeline for |
| * the duration of this interaction. |
| */ |
| public void requestDelegating() { |
| validateTransitionRequest(); |
| final IAccessibilityServiceConnection connection = |
| AccessibilityInteractionClient.getInstance() |
| .getConnection(mService.getConnectionId()); |
| if (connection != null) { |
| try { |
| connection.requestDelegating(mDisplayId); |
| } catch (RemoteException re) { |
| throw new RuntimeException(re); |
| } |
| mStateChangeRequested = true; |
| } |
| } |
| |
| /** |
| * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If |
| * {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least |
| * one callback has been added, this function tells the framework to perform a click. |
| * The framework will first try to perform |
| * {@link AccessibilityNodeInfo.AccessibilityAction#ACTION_CLICK} on the item with |
| * accessibility focus. If that fails, the framework will simulate a click using motion events |
| * on the last location to have accessibility focus. |
| */ |
| public void performClick() { |
| final IAccessibilityServiceConnection connection = |
| AccessibilityInteractionClient.getInstance() |
| .getConnection(mService.getConnectionId()); |
| if (connection != null) { |
| try { |
| connection.onDoubleTap(mDisplayId); |
| } catch (RemoteException re) { |
| throw new RuntimeException(re); |
| } |
| } |
| } |
| |
| /** |
| * If {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} and {@link If |
| * {@link AccessibilityServiceInfo#FLAG_REQUEST_TOUCH_EXPLORATION_MODE} is enabled and at least |
| * one callback has been added, this function tells the framework to perform a long click. |
| * The framework will simulate a long click using motion events on the last location with |
| * accessibility focus and will delegate any movements to the rest of the input pipeline. This |
| * allows a user to double-tap and hold to trigger a drag and then execute that drag by moving |
| * their finger. |
| */ |
| public void performLongClickAndStartDrag() { |
| final IAccessibilityServiceConnection connection = |
| AccessibilityInteractionClient.getInstance() |
| .getConnection(mService.getConnectionId()); |
| if (connection != null) { |
| try { |
| connection.onDoubleTapAndHold(mDisplayId); |
| } catch (RemoteException re) { |
| throw new RuntimeException(re); |
| } |
| } |
| } |
| |
| private void validateTransitionRequest() { |
| if (!mServiceDetectsGestures || mCallbacks.size() == 0) { |
| throw new IllegalStateException( |
| "State transitions are not allowed without first adding a callback."); |
| } |
| if ((mState == STATE_DELEGATING || mState == STATE_TOUCH_EXPLORING)) { |
| throw new IllegalStateException( |
| "State transition requests are not allowed in " + stateToString(mState)); |
| } |
| } |
| |
| /** @return the maximum number of pointers that this display will accept. */ |
| public int getMaxPointerCount() { |
| return MAX_POINTER_COUNT; |
| } |
| |
| /** @return the display id associated with this controller. */ |
| public int getDisplayId() { |
| return mDisplayId; |
| } |
| |
| /** |
| * @return the current state of this controller. |
| * @see TouchInteractionController#STATE_CLEAR |
| * @see TouchInteractionController#STATE_DELEGATING |
| * @see TouchInteractionController#STATE_DRAGGING |
| * @see TouchInteractionController#STATE_TOUCH_EXPLORING |
| */ |
| public int getState() { |
| synchronized (mLock) { |
| return mState; |
| } |
| } |
| |
| /** Returns a string representation of the specified state. */ |
| @NonNull |
| public static String stateToString(int state) { |
| switch (state) { |
| case STATE_CLEAR: |
| return "STATE_CLEAR"; |
| case STATE_TOUCH_INTERACTING: |
| return "STATE_TOUCH_INTERACTING"; |
| case STATE_TOUCH_EXPLORING: |
| return "STATE_TOUCH_EXPLORING"; |
| case STATE_DRAGGING: |
| return "STATE_DRAGGING"; |
| case STATE_DELEGATING: |
| return "STATE_DELEGATING"; |
| default: |
| return "Unknown state: " + state; |
| } |
| } |
| |
| /** callbacks allow services to receive motion events and state change updates. */ |
| public interface Callback { |
| /** |
| * Called when the framework has sent a motion event to the service. |
| * |
| * @param event the event being passed to the service. |
| */ |
| void onMotionEvent(@NonNull MotionEvent event); |
| |
| /** |
| * Called when the state of motion event dispatch for this display has changed. |
| * |
| * @param state the new state of motion event dispatch. |
| * @see TouchInteractionController#STATE_CLEAR |
| * @see TouchInteractionController#STATE_DELEGATING |
| * @see TouchInteractionController#STATE_DRAGGING |
| * @see TouchInteractionController#STATE_TOUCH_EXPLORING |
| */ |
| void onStateChanged(@State int state); |
| } |
| } |