blob: 66807aeb66298e1ecb4307df2f67790dc36b0e30 [file] [log] [blame]
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.inputmethod;
import android.Manifest;
import android.annotation.AnyThread;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.annotation.RequiresPermission;
import android.annotation.UiThread;
import android.content.ComponentName;
import android.content.Context;
import android.content.pm.PackageManagerInternal;
import android.hardware.input.InputManager;
import android.hardware.input.InputManagerGlobal;
import android.os.Handler;
import android.os.IBinder;
import android.os.Looper;
import android.text.TextUtils;
import android.util.Slog;
import android.view.BatchedInputEventReceiver;
import android.view.Choreographer;
import android.view.Display;
import android.view.InputChannel;
import android.view.InputEvent;
import android.view.InputEventReceiver;
import android.view.MotionEvent;
import android.view.PointerIcon;
import android.view.SurfaceControl;
import android.view.View;
import android.view.inputmethod.InputMethodManager;
import com.android.server.LocalServices;
import com.android.server.input.InputManagerInternal;
import com.android.server.wm.WindowManagerInternal;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.OptionalInt;
// TODO(b/210039666): See if we can make this class thread-safe.
final class HandwritingModeController {
public static final String TAG = HandwritingModeController.class.getSimpleName();
static final boolean DEBUG = false;
// Use getHandwritingBufferSize() and not this value directly.
private static final int EVENT_BUFFER_SIZE = 100;
// A longer event buffer used for handwriting delegation
// TODO(b/210039666): make this device touch sampling rate dependent.
// Use getHandwritingBufferSize() and not this value directly.
private static final int LONG_EVENT_BUFFER_SIZE = EVENT_BUFFER_SIZE * 20;
private static final long HANDWRITING_DELEGATION_IDLE_TIMEOUT_MS = 3000;
private final Context mContext;
// This must be the looper for the UiThread.
private final Looper mLooper;
private final InputManagerInternal mInputManagerInternal;
private final WindowManagerInternal mWindowManagerInternal;
private final PackageManagerInternal mPackageManagerInternal;
private ArrayList<MotionEvent> mHandwritingBuffer;
private InputEventReceiver mHandwritingEventReceiver;
private Runnable mInkWindowInitRunnable;
private boolean mRecordingGesture;
private int mCurrentDisplayId;
// when set, package names are used for handwriting delegation.
private @Nullable String mDelegatePackageName;
private @Nullable String mDelegatorPackageName;
private boolean mDelegatorFromDefaultHomePackage;
private Runnable mDelegationIdleTimeoutRunnable;
private Handler mDelegationIdleTimeoutHandler;
private HandwritingEventReceiverSurface mHandwritingSurface;
private int mCurrentRequestId;
@AnyThread
HandwritingModeController(Context context, Looper uiThreadLooper,
Runnable inkWindowInitRunnable) {
mContext = context;
mLooper = uiThreadLooper;
mCurrentDisplayId = Display.INVALID_DISPLAY;
mInputManagerInternal = LocalServices.getService(InputManagerInternal.class);
mWindowManagerInternal = LocalServices.getService(WindowManagerInternal.class);
mPackageManagerInternal = LocalServices.getService(PackageManagerInternal.class);
mCurrentRequestId = 0;
mInkWindowInitRunnable = inkWindowInitRunnable;
}
/**
* Initializes the handwriting spy on the given displayId.
*
* This must be called from the UI Thread because it will start processing events using an
* InputEventReceiver that batches events according to the current thread's Choreographer.
*/
@UiThread
void initializeHandwritingSpy(int displayId) {
// When resetting, reuse resources if we are reinitializing on the same display.
reset(displayId == mCurrentDisplayId);
mCurrentDisplayId = displayId;
if (mHandwritingBuffer == null) {
mHandwritingBuffer = new ArrayList<>(getHandwritingBufferSize());
}
if (DEBUG) Slog.d(TAG, "Initializing handwriting spy monitor for display: " + displayId);
final String name = "stylus-handwriting-event-receiver-" + displayId;
final InputChannel channel = mInputManagerInternal.createInputChannel(name);
Objects.requireNonNull(channel, "Failed to create input channel");
final SurfaceControl surface =
mHandwritingSurface != null ? mHandwritingSurface.getSurface()
: mWindowManagerInternal.getHandwritingSurfaceForDisplay(displayId);
if (surface == null) {
Slog.e(TAG, "Failed to create input surface");
return;
}
mHandwritingSurface = new HandwritingEventReceiverSurface(
name, displayId, surface, channel);
// Use a dup of the input channel so that event processing can be paused by disposing the
// event receiver without causing a fd hangup.
mHandwritingEventReceiver = new BatchedInputEventReceiver.SimpleBatchedInputEventReceiver(
channel.dup(), mLooper, Choreographer.getInstance(), this::onInputEvent);
mCurrentRequestId++;
}
OptionalInt getCurrentRequestId() {
if (mHandwritingSurface == null) {
Slog.e(TAG, "Cannot get requestId: Handwriting was not initialized.");
return OptionalInt.empty();
}
return OptionalInt.of(mCurrentRequestId);
}
boolean isStylusGestureOngoing() {
return mRecordingGesture;
}
boolean hasOngoingStylusHandwritingSession() {
return mHandwritingSurface != null && mHandwritingSurface.isIntercepting();
}
/**
* Prepare delegation of stylus handwriting to a different editor
* @see InputMethodManager#prepareStylusHandwritingDelegation(View, String)
*/
void prepareStylusHandwritingDelegation(
int userId, @NonNull String delegatePackageName, @NonNull String delegatorPackageName) {
mDelegatePackageName = delegatePackageName;
mDelegatorPackageName = delegatorPackageName;
mDelegatorFromDefaultHomePackage = false;
// mDelegatorFromDefaultHomeActivity is only used in the cross-package delegation case.
// For same-package delegation, it doesn't need to be checked.
if (!delegatorPackageName.equals(delegatePackageName)) {
ComponentName defaultHomeActivity =
mPackageManagerInternal.getDefaultHomeActivity(userId);
if (defaultHomeActivity != null) {
mDelegatorFromDefaultHomePackage =
delegatorPackageName.equals(defaultHomeActivity.getPackageName());
}
}
if (mHandwritingBuffer == null) {
mHandwritingBuffer = new ArrayList<>(getHandwritingBufferSize());
} else {
mHandwritingBuffer.ensureCapacity(getHandwritingBufferSize());
}
scheduleHandwritingDelegationTimeout();
}
@Nullable String getDelegatePackageName() {
return mDelegatePackageName;
}
@Nullable String getDelegatorPackageName() {
return mDelegatorPackageName;
}
boolean isDelegatorFromDefaultHomePackage() {
return mDelegatorFromDefaultHomePackage;
}
private void scheduleHandwritingDelegationTimeout() {
if (mDelegationIdleTimeoutHandler == null) {
mDelegationIdleTimeoutHandler = new Handler(mLooper);
} else {
mDelegationIdleTimeoutHandler.removeCallbacks(mDelegationIdleTimeoutRunnable);
}
mDelegationIdleTimeoutRunnable = () -> {
Slog.d(TAG, "Stylus handwriting delegation idle timed-out.");
clearPendingHandwritingDelegation();
if (mHandwritingBuffer != null) {
mHandwritingBuffer.forEach(MotionEvent::recycle);
mHandwritingBuffer.clear();
mHandwritingBuffer.trimToSize();
mHandwritingBuffer.ensureCapacity(getHandwritingBufferSize());
}
};
mDelegationIdleTimeoutHandler.postDelayed(
mDelegationIdleTimeoutRunnable, HANDWRITING_DELEGATION_IDLE_TIMEOUT_MS);
}
private int getHandwritingBufferSize() {
if (mDelegatePackageName != null && mDelegatorPackageName != null) {
return LONG_EVENT_BUFFER_SIZE;
}
return EVENT_BUFFER_SIZE;
}
/**
* Clear any pending handwriting delegation info.
*/
void clearPendingHandwritingDelegation() {
if (DEBUG) {
Slog.d(TAG, "clearPendingHandwritingDelegation");
}
if (mDelegationIdleTimeoutHandler != null) {
mDelegationIdleTimeoutHandler.removeCallbacks(mDelegationIdleTimeoutRunnable);
mDelegationIdleTimeoutHandler = null;
}
mDelegationIdleTimeoutRunnable = null;
mDelegatorPackageName = null;
mDelegatePackageName = null;
mDelegatorFromDefaultHomePackage = false;
}
/**
* Starts a {@link HandwritingSession} to transfer to the IME.
*
* This must be called from the UI Thread to avoid race conditions between processing more
* input events and disposing the input event receiver.
* @return the handwriting session to send to the IME, or null if the request was invalid.
*/
@RequiresPermission(Manifest.permission.MONITOR_INPUT)
@UiThread
@Nullable
HandwritingSession startHandwritingSession(
int requestId, int imePid, int imeUid, IBinder focusedWindowToken) {
clearPendingHandwritingDelegation();
if (mHandwritingSurface == null) {
Slog.e(TAG, "Cannot start handwriting session: Handwriting was not initialized.");
return null;
}
if (requestId != mCurrentRequestId) {
Slog.e(TAG, "Cannot start handwriting session: Invalid request id: " + requestId);
return null;
}
if (!mRecordingGesture || mHandwritingBuffer.isEmpty()) {
Slog.e(TAG, "Cannot start handwriting session: No stylus gesture is being recorded.");
return null;
}
Objects.requireNonNull(mHandwritingEventReceiver,
"Handwriting session was already transferred to IME.");
final MotionEvent downEvent = mHandwritingBuffer.get(0);
assert (downEvent.getActionMasked() == MotionEvent.ACTION_DOWN);
if (!mWindowManagerInternal.isPointInsideWindow(
focusedWindowToken, mCurrentDisplayId, downEvent.getRawX(), downEvent.getRawY())) {
Slog.e(TAG, "Cannot start handwriting session: "
+ "Stylus gesture did not start inside the focused window.");
return null;
}
if (DEBUG) Slog.d(TAG, "Starting handwriting session in display: " + mCurrentDisplayId);
InputManagerGlobal.getInstance()
.pilferPointers(mHandwritingSurface.getInputChannel().getToken());
// Stop processing more events.
mHandwritingEventReceiver.dispose();
mHandwritingEventReceiver = null;
mRecordingGesture = false;
if (mHandwritingSurface.isIntercepting()) {
throw new IllegalStateException(
"Handwriting surface should not be already intercepting.");
}
mHandwritingSurface.startIntercepting(imePid, imeUid);
// Unset the pointer icon for the stylus in case the app had set it.
if (com.android.input.flags.Flags.enablePointerChoreographer()) {
Objects.requireNonNull(mContext.getSystemService(InputManager.class)).setPointerIcon(
PointerIcon.getSystemIcon(mContext, PointerIcon.TYPE_NOT_SPECIFIED),
downEvent.getDisplayId(), downEvent.getDeviceId(), downEvent.getPointerId(0),
mHandwritingSurface.getInputChannel().getToken());
} else {
InputManagerGlobal.getInstance().setPointerIconType(PointerIcon.TYPE_NOT_SPECIFIED);
}
return new HandwritingSession(mCurrentRequestId, mHandwritingSurface.getInputChannel(),
mHandwritingBuffer);
}
/**
* Reset the current handwriting session without initializing another session.
*
* This must be called from UI Thread to avoid race conditions between processing more input
* events and disposing the input event receiver.
*/
@UiThread
void reset() {
reset(false /* reinitializing */);
}
void setInkWindowInitializer(Runnable inkWindowInitializer) {
mInkWindowInitRunnable = inkWindowInitializer;
}
private void reset(boolean reinitializing) {
if (mHandwritingEventReceiver != null) {
mHandwritingEventReceiver.dispose();
mHandwritingEventReceiver = null;
}
if (mHandwritingBuffer != null) {
mHandwritingBuffer.forEach(MotionEvent::recycle);
mHandwritingBuffer.clear();
if (!reinitializing) {
mHandwritingBuffer = null;
}
}
if (mHandwritingSurface != null) {
mHandwritingSurface.getInputChannel().dispose();
if (!reinitializing) {
mHandwritingSurface.remove();
mHandwritingSurface = null;
}
}
clearPendingHandwritingDelegation();
mRecordingGesture = false;
}
private boolean onInputEvent(InputEvent ev) {
if (mHandwritingEventReceiver == null) {
throw new IllegalStateException(
"Input Event should not be processed when IME has the spy channel.");
}
if (!(ev instanceof MotionEvent)) {
Slog.wtf(TAG, "Received non-motion event in stylus monitor.");
return false;
}
final MotionEvent event = (MotionEvent) ev;
if (!event.isStylusPointer()) {
return false;
}
if (event.getDisplayId() != mCurrentDisplayId) {
Slog.wtf(TAG, "Received stylus event associated with the incorrect display.");
return false;
}
onStylusEvent(event);
return true;
}
private void onStylusEvent(MotionEvent event) {
final int action = event.getActionMasked();
if (mInkWindowInitRunnable != null && (action == MotionEvent.ACTION_HOVER_ENTER
|| event.getAction() == MotionEvent.ACTION_HOVER_ENTER)) {
// Ask IMMS to make ink window ready.
mInkWindowInitRunnable.run();
mInkWindowInitRunnable = null;
return;
} else if (event.isHoverEvent()) {
// Hover events need not be recorded to buffer.
return;
}
// If handwriting delegation is ongoing, don't clear the buffer so that multiple strokes
// can be buffered across windows.
if (TextUtils.isEmpty(mDelegatePackageName)
&& (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL)) {
mRecordingGesture = false;
mHandwritingBuffer.clear();
return;
}
if (action == MotionEvent.ACTION_DOWN) {
mRecordingGesture = true;
}
if (!mRecordingGesture) {
return;
}
if (mHandwritingBuffer.size() >= getHandwritingBufferSize()) {
if (DEBUG) {
Slog.w(TAG, "Current gesture exceeds the buffer capacity."
+ " The rest of the gesture will not be recorded.");
}
mRecordingGesture = false;
return;
}
mHandwritingBuffer.add(MotionEvent.obtain(event));
}
static final class HandwritingSession {
private final int mRequestId;
private final InputChannel mHandwritingChannel;
private final List<MotionEvent> mRecordedEvents;
private HandwritingSession(int requestId, InputChannel handwritingChannel,
List<MotionEvent> recordedEvents) {
mRequestId = requestId;
mHandwritingChannel = handwritingChannel;
mRecordedEvents = recordedEvents;
}
int getRequestId() {
return mRequestId;
}
InputChannel getHandwritingChannel() {
return mHandwritingChannel;
}
List<MotionEvent> getRecordedEvents() {
return mRecordedEvents;
}
}
}