blob: 5d2d5826f227dd15fd14c0dfc782f3f2a22575fd [file] [log] [blame]
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.server.wm;
import static android.view.ContentRecordingSession.RECORD_CONTENT_DISPLAY;
import static android.view.ContentRecordingSession.RECORD_CONTENT_TASK;
import static com.android.internal.protolog.ProtoLogGroup.WM_DEBUG_CONTENT_RECORDING;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.res.Configuration;
import android.graphics.Point;
import android.graphics.Rect;
import android.os.IBinder;
import android.provider.DeviceConfig;
import android.view.ContentRecordingSession;
import android.view.Display;
import android.view.SurfaceControl;
import com.android.internal.annotations.VisibleForTesting;
import com.android.internal.protolog.common.ProtoLog;
/**
* Manages content recording for a particular {@link DisplayContent}.
*/
final class ContentRecorder {
/**
* The key for accessing the device config that controls if task recording is supported.
*/
@VisibleForTesting static final String KEY_RECORD_TASK_FEATURE = "record_task_content";
/**
* The display content this class is handling recording for.
*/
@NonNull
private final DisplayContent mDisplayContent;
/**
* The session for content recording, or null if this DisplayContent is not being used for
* recording.
*/
private ContentRecordingSession mContentRecordingSession = null;
/**
* The WindowContainer for the level of the hierarchy to record.
*/
@Nullable private WindowContainer mRecordedWindowContainer = null;
/**
* The surface for recording the contents of this hierarchy, or null if content recording is
* temporarily disabled.
*/
@Nullable private SurfaceControl mRecordedSurface = null;
/**
* The last bounds of the region to record.
*/
@Nullable private Rect mLastRecordedBounds = null;
ContentRecorder(@NonNull DisplayContent displayContent) {
mDisplayContent = displayContent;
}
/**
* Sets the incoming recording session. Should only be used when starting to record on
* this display; stopping recording is handled separately when the display is destroyed.
*
* @param session the new session indicating recording will begin on this display.
*/
void setContentRecordingSession(@Nullable ContentRecordingSession session) {
mContentRecordingSession = session;
}
/**
* Returns {@code true} if this DisplayContent is currently recording.
*/
boolean isCurrentlyRecording() {
return mContentRecordingSession != null && mRecordedSurface != null;
}
/**
* Start recording if this DisplayContent no longer has content. Stop recording if it now
* has content or the display is not on.
*/
@VisibleForTesting void updateRecording() {
if (isCurrentlyRecording() && (mDisplayContent.getLastHasContent()
|| mDisplayContent.getDisplay().getState() == Display.STATE_OFF)) {
pauseRecording();
} else {
// Display no longer has content, or now has a surface to write to, so try to start
// recording.
startRecordingIfNeeded();
}
}
/**
* Handle a configuration change on the display content, and resize recording if needed.
* @param lastOrientation the prior orientation of the configuration
*/
void onConfigurationChanged(@Configuration.Orientation int lastOrientation) {
// Update surface for MediaProjection, if this DisplayContent is being used for recording.
if (isCurrentlyRecording() && mLastRecordedBounds != null) {
// Recording has already begun, but update recording since the display is now on.
if (mRecordedWindowContainer == null) {
ProtoLog.v(WM_DEBUG_CONTENT_RECORDING,
"Unexpectedly null window container; unable to update recording for "
+ "display %d",
mDisplayContent.getDisplayId());
return;
}
ProtoLog.v(WM_DEBUG_CONTENT_RECORDING,
"Display %d was already recording, so apply transformations if necessary",
mDisplayContent.getDisplayId());
// Retrieve the size of the region to record, and continue with the update
// if the bounds or orientation has changed.
final Rect recordedContentBounds = mRecordedWindowContainer.getBounds();
int recordedContentOrientation = mRecordedWindowContainer.getOrientation();
if (!mLastRecordedBounds.equals(recordedContentBounds)
|| lastOrientation != recordedContentOrientation) {
Point surfaceSize = fetchSurfaceSizeIfPresent();
if (surfaceSize != null) {
ProtoLog.v(WM_DEBUG_CONTENT_RECORDING,
"Going ahead with updating recording for display %d to new "
+ "bounds %s and/or orientation %d.",
mDisplayContent.getDisplayId(), recordedContentBounds,
recordedContentOrientation);
updateMirroredSurface(mDisplayContent.mWmService.mTransactionFactory.get(),
recordedContentBounds, surfaceSize);
} else {
// If the surface removed, do nothing. We will handle this via onDisplayChanged
// (the display will be off if the surface is removed).
ProtoLog.v(WM_DEBUG_CONTENT_RECORDING,
"Unable to update recording for display %d to new bounds %s"
+ " and/or orientation %d, since the surface is not available.",
mDisplayContent.getDisplayId(), recordedContentBounds,
recordedContentOrientation);
}
}
}
}
/**
* Pauses recording on this display content. Note the session does not need to be updated,
* since recording can be resumed still.
*/
void pauseRecording() {
if (mRecordedSurface == null) {
return;
}
ProtoLog.v(WM_DEBUG_CONTENT_RECORDING,
"Display %d has content (%b) so pause recording", mDisplayContent.getDisplayId(),
mDisplayContent.getLastHasContent());
// If the display is not on and it is a virtual display, then it no longer has an
// associated surface to write output to.
// If the display now has content, stop mirroring to it.
mDisplayContent.mWmService.mTransactionFactory.get()
// Remove the reference to mMirroredSurface, to clean up associated memory.
.remove(mRecordedSurface)
// Reparent the SurfaceControl of this DisplayContent back to mSurfaceControl,
// to allow content to be added to it. This allows this DisplayContent to stop
// mirroring and show content normally.
.reparent(mDisplayContent.getWindowingLayer(), mDisplayContent.getSurfaceControl())
.reparent(mDisplayContent.getOverlayLayer(), mDisplayContent.getSurfaceControl())
.apply();
// Pause mirroring by destroying the reference to the mirrored layer.
mRecordedSurface = null;
// Do not un-set the token, in case content is removed and recording should begin again.
}
/**
* Stops recording on this DisplayContent, and updates the session details.
*/
void remove() {
if (mRecordedSurface != null) {
// Do not wait for the mirrored surface to be garbage collected, but clean up
// immediately.
mDisplayContent.mWmService.mTransactionFactory.get().remove(mRecordedSurface).apply();
mRecordedSurface = null;
clearContentRecordingSession();
// Do not need to force remove the VirtualDisplay; this is handled by the media
// projection service.
}
}
/**
* Removes both the local cache and WM Service view of the current session, to stop the session
* on this display.
*/
private void clearContentRecordingSession() {
// Update the cached session state first, since updating the service will result in always
// returning to this instance to update recording state.
mContentRecordingSession = null;
mDisplayContent.mWmService.mContentRecordingController.setContentRecordingSessionLocked(
null, mDisplayContent.mWmService);
}
/**
* Start recording to this DisplayContent if it does not have its own content. Captures the
* content of a WindowContainer indicated by a WindowToken. If unable to start recording, falls
* back to original MediaProjection approach.
*/
private void startRecordingIfNeeded() {
// Only record if this display does not have its own content, is not recording already,
// and if this display is on (it has a surface to write output to).
if (mDisplayContent.getLastHasContent() || isCurrentlyRecording()
|| mDisplayContent.getDisplay().getState() == Display.STATE_OFF
|| mContentRecordingSession == null) {
return;
}
mRecordedWindowContainer = retrieveRecordedWindowContainer();
if (mRecordedWindowContainer == null) {
// Either the token is missing, or the window associated with the token is missing.
// Error has already been handled, so just leave.
return;
}
final Point surfaceSize = fetchSurfaceSizeIfPresent();
if (surfaceSize == null) {
ProtoLog.v(WM_DEBUG_CONTENT_RECORDING,
"Unable to start recording for display %d since the surface is not "
+ "available.",
mDisplayContent.getDisplayId());
return;
}
ProtoLog.v(WM_DEBUG_CONTENT_RECORDING,
"Display %d has no content and is on, so start recording for state %d",
mDisplayContent.getDisplayId(), mDisplayContent.getDisplay().getState());
// Create a mirrored hierarchy for the SurfaceControl of the DisplayArea to capture.
mRecordedSurface = SurfaceControl.mirrorSurface(
mRecordedWindowContainer.getSurfaceControl());
SurfaceControl.Transaction transaction =
mDisplayContent.mWmService.mTransactionFactory.get()
// Set the mMirroredSurface's parent to the root SurfaceControl for this
// DisplayContent. This brings the new mirrored hierarchy under this
// DisplayContent,
// so SurfaceControl will write the layers of this hierarchy to the
// output surface
// provided by the app.
.reparent(mRecordedSurface, mDisplayContent.getSurfaceControl())
// Reparent the SurfaceControl of this DisplayContent to null, to prevent
// content
// being added to it. This ensures that no app launched explicitly on the
// VirtualDisplay will show up as part of the mirrored content.
.reparent(mDisplayContent.getWindowingLayer(), null)
.reparent(mDisplayContent.getOverlayLayer(), null);
// Retrieve the size of the DisplayArea to mirror.
updateMirroredSurface(transaction, mRecordedWindowContainer.getBounds(), surfaceSize);
// No need to clean up. In SurfaceFlinger, parents hold references to their children. The
// mirrored SurfaceControl is alive since the parent DisplayContent SurfaceControl is
// holding a reference to it. Therefore, the mirrored SurfaceControl will be cleaned up
// when the VirtualDisplay is destroyed - which will clean up this DisplayContent.
}
/**
* Retrieves the {@link WindowContainer} for the level of the hierarchy to start recording,
* indicated by the {@link #mContentRecordingSession}. Performs any error handling and state
* updates necessary if the {@link WindowContainer} could not be retrieved.
* {@link #mContentRecordingSession} must be non-null.
*
* @return a {@link WindowContainer} to record, or {@code null} if an error was encountered. The
* error is logged and any cleanup is handled.
*/
@Nullable
private WindowContainer retrieveRecordedWindowContainer() {
final int contentToRecord = mContentRecordingSession.getContentToRecord();
// Given the WindowToken of the region to record, retrieve the associated
// SurfaceControl.
final IBinder tokenToRecord = mContentRecordingSession.getTokenToRecord();
if (tokenToRecord == null) {
handleStartRecordingFailed();
ProtoLog.v(WM_DEBUG_CONTENT_RECORDING,
"Unable to start recording due to null token for display %d",
mDisplayContent.getDisplayId());
return null;
}
switch (contentToRecord) {
case RECORD_CONTENT_DISPLAY:
final WindowContainer wc =
mDisplayContent.mWmService.mWindowContextListenerController.getContainer(
tokenToRecord);
if (wc == null) {
// Un-set the window token to record for this VirtualDisplay. Fall back to
// Display stack capture for the entire display.
mDisplayContent.mWmService.mDisplayManagerInternal.setWindowManagerMirroring(
mDisplayContent.getDisplayId(), false);
handleStartRecordingFailed();
ProtoLog.v(WM_DEBUG_CONTENT_RECORDING,
"Unable to retrieve window container to start recording for "
+ "display %d", mDisplayContent.getDisplayId());
return null;
}
// TODO(206461622) Migrate to using the RootDisplayArea
return wc.getDisplayContent();
case RECORD_CONTENT_TASK:
if (!DeviceConfig.getBoolean(DeviceConfig.NAMESPACE_WINDOW_MANAGER,
KEY_RECORD_TASK_FEATURE, false)) {
handleStartRecordingFailed();
ProtoLog.v(WM_DEBUG_CONTENT_RECORDING,
"Unable to record task since feature is disabled %d",
mDisplayContent.getDisplayId());
return null;
}
Task taskToRecord = WindowContainer.fromBinder(tokenToRecord).asTask();
if (taskToRecord == null) {
handleStartRecordingFailed();
ProtoLog.v(WM_DEBUG_CONTENT_RECORDING,
"Unable to retrieve task to start recording for "
+ "display %d", mDisplayContent.getDisplayId());
}
return taskToRecord;
default:
// Not a valid region, or recording is disabled, so fall back to Display stack
// capture for the entire display.
handleStartRecordingFailed();
ProtoLog.v(WM_DEBUG_CONTENT_RECORDING,
"Unable to start recording due to invalid region for display %d",
mDisplayContent.getDisplayId());
return null;
}
}
/**
* Exit this recording session.
* <p>
* If this is a task session, tear down the recording entirely. Do not fall back
* to recording the entire display on the display stack; this would surprise the user
* given they selected task capture.
* </p><p>
* If this is a display session, just stop recording by layer mirroring. Fall back to recording
* from the display stack.
* </p>
*/
private void handleStartRecordingFailed() {
final boolean shouldExitTaskRecording = mContentRecordingSession != null
&& mContentRecordingSession.getContentToRecord() == RECORD_CONTENT_TASK;
if (shouldExitTaskRecording) {
// Clean up the cached session first, since tearing down the display will generate
// display
// events which will trickle back to here.
clearContentRecordingSession();
tearDownVirtualDisplay();
} else {
clearContentRecordingSession();
}
}
/**
* Ensure recording does not fall back to the display stack; ensure the recording is stopped
* and the client notified by tearing down the virtual display.
*/
private void tearDownVirtualDisplay() {
// TODO(b/219761722) Clean up the VirtualDisplay if task mirroring fails
}
/**
* Apply transformations to the mirrored surface to ensure the captured contents are scaled to
* fit and centred in the output surface.
*
* @param transaction the transaction to include transformations of mMirroredSurface
* to. Transaction is not applied before returning.
* @param recordedContentBounds bounds of the content to record to the surface provided by
* the app.
* @param surfaceSize the default size of the surface to write the display area
* content to
*/
@VisibleForTesting void updateMirroredSurface(SurfaceControl.Transaction transaction,
Rect recordedContentBounds, Point surfaceSize) {
// Calculate the scale to apply to the root mirror SurfaceControl to fit the size of the
// output surface.
float scaleX = surfaceSize.x / (float) recordedContentBounds.width();
float scaleY = surfaceSize.y / (float) recordedContentBounds.height();
float scale = Math.min(scaleX, scaleY);
int scaledWidth = Math.round(scale * (float) recordedContentBounds.width());
int scaledHeight = Math.round(scale * (float) recordedContentBounds.height());
// Calculate the shift to apply to the root mirror SurfaceControl to centre the mirrored
// contents in the output surface.
int shiftedX = 0;
if (scaledWidth != surfaceSize.x) {
shiftedX = (surfaceSize.x - scaledWidth) / 2;
}
int shiftedY = 0;
if (scaledHeight != surfaceSize.y) {
shiftedY = (surfaceSize.y - scaledHeight) / 2;
}
transaction
// Crop the area to capture to exclude the 'extra' wallpaper that is used
// for parallax (b/189930234).
.setWindowCrop(mRecordedSurface, recordedContentBounds.width(),
recordedContentBounds.height())
// Scale the root mirror SurfaceControl, based upon the size difference between the
// source (DisplayArea to capture) and output (surface the app reads images from).
.setMatrix(mRecordedSurface, scale, 0 /* dtdx */, 0 /* dtdy */, scale)
// Position needs to be updated when the mirrored DisplayArea has changed, since
// the content will no longer be centered in the output surface.
.setPosition(mRecordedSurface, shiftedX /* x */, shiftedY /* y */)
.apply();
mLastRecordedBounds = new Rect(recordedContentBounds);
}
/**
* Returns a non-null {@link Point} if the surface is present, or null otherwise
*/
private Point fetchSurfaceSizeIfPresent() {
// Retrieve the default size of the surface the app provided to
// MediaProjection#createVirtualDisplay. Note the app is the consumer of the surface,
// since it reads out buffers from the surface, and SurfaceFlinger is the producer since
// it writes the mirrored layers to the buffers.
Point surfaceSize =
mDisplayContent.mWmService.mDisplayManagerInternal.getDisplaySurfaceDefaultSize(
mDisplayContent.getDisplayId());
if (surfaceSize == null) {
// Layer mirroring started with a null surface, so do not apply any transformations yet.
// State of virtual display will change to 'ON' when the surface is set.
// will get event DISPLAY_DEVICE_EVENT_CHANGED
ProtoLog.v(WM_DEBUG_CONTENT_RECORDING,
"Provided surface for recording on display %d is not present, so do not"
+ " update the surface",
mDisplayContent.getDisplayId());
return null;
}
return surfaceSize;
}
}