| /* |
| * Copyright (C) 2022 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package com.android.server.voiceinteraction; |
| |
| import static android.Manifest.permission.CAPTURE_AUDIO_HOTWORD; |
| import static android.Manifest.permission.LOG_COMPAT_CHANGE; |
| import static android.Manifest.permission.READ_COMPAT_CHANGE_CONFIG; |
| import static android.Manifest.permission.RECORD_AUDIO; |
| import static android.service.attention.AttentionService.PROXIMITY_UNKNOWN; |
| import static android.service.voice.HotwordDetectionService.AUDIO_SOURCE_EXTERNAL; |
| import static android.service.voice.HotwordDetectionService.ENABLE_PROXIMITY_RESULT; |
| import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_SUCCESS; |
| import static android.service.voice.HotwordDetectionService.INITIALIZATION_STATUS_UNKNOWN; |
| import static android.service.voice.HotwordDetectionService.KEY_INITIALIZATION_STATUS; |
| |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_ERROR; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_NO_VALUE; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_OVER_MAX_CUSTOM_VALUE; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; |
| import static com.android.internal.util.FrameworkStatsLog.HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; |
| import static com.android.server.voiceinteraction.HotwordDetectionConnection.ENFORCE_HOTWORD_PHRASE_ID; |
| import static com.android.server.voiceinteraction.SoundTriggerSessionPermissionsDecorator.enforcePermissionForPreflight; |
| |
| import android.annotation.NonNull; |
| import android.annotation.Nullable; |
| import android.annotation.RequiresPermission; |
| import android.app.AppOpsManager; |
| import android.app.compat.CompatChanges; |
| import android.attention.AttentionManagerInternal; |
| import android.content.Context; |
| import android.content.PermissionChecker; |
| import android.hardware.soundtrigger.SoundTrigger; |
| import android.media.AudioFormat; |
| import android.media.permission.Identity; |
| import android.media.permission.PermissionUtil; |
| import android.os.Binder; |
| import android.os.Bundle; |
| import android.os.IBinder; |
| import android.os.IRemoteCallback; |
| import android.os.ParcelFileDescriptor; |
| import android.os.PersistableBundle; |
| import android.os.RemoteException; |
| import android.os.SharedMemory; |
| import android.service.voice.HotwordDetectedResult; |
| import android.service.voice.HotwordDetectionService; |
| import android.service.voice.HotwordDetectionServiceFailure; |
| import android.service.voice.HotwordDetector; |
| import android.service.voice.HotwordRejectedResult; |
| import android.service.voice.IDspHotwordDetectionCallback; |
| import android.service.voice.IMicrophoneHotwordDetectionVoiceInteractionCallback; |
| import android.text.TextUtils; |
| import android.util.Pair; |
| import android.util.Slog; |
| |
| import com.android.internal.annotations.GuardedBy; |
| import com.android.internal.app.IHotwordRecognitionStatusCallback; |
| import com.android.internal.infra.AndroidFuture; |
| import com.android.server.LocalServices; |
| |
| import java.io.Closeable; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.OutputStream; |
| import java.io.PrintWriter; |
| import java.time.Duration; |
| import java.time.Instant; |
| import java.util.concurrent.Executor; |
| import java.util.concurrent.Executors; |
| import java.util.concurrent.ScheduledExecutorService; |
| import java.util.concurrent.TimeUnit; |
| import java.util.concurrent.TimeoutException; |
| import java.util.concurrent.atomic.AtomicBoolean; |
| |
| /** |
| * A class that provides sandboxed detector to communicate with the {@link |
| * HotwordDetectionService} and {@link VisualQueryDetectionService}. |
| * |
| * Trusted hotword detectors such as {@link SoftwareHotwordDetector} and |
| * {@link AlwaysOnHotwordDetector} will leverage this class to communitcate with |
| * {@link HotwordDetectionService}; similarly, {@link VisualQueryDetector} will communicate with |
| * {@link VisualQueryDetectionService}. |
| * |
| * This class provides the methods to do initialization with the {@link HotwordDetectionService} and |
| * {@link VisualQueryDetectionService} handles external source detection for |
| * {@link HotwordDetectionService}. It also provides the methods to check if we can egress the data |
| * from the {@link HotwordDetectionService} and {@link VisualQueryDetectionService}. |
| * |
| * The subclass should override the {@link #informRestartProcessLocked()} to handle the trusted |
| * process restart. |
| */ |
| abstract class DetectorSession { |
| private static final String TAG = "DetectorSession"; |
| static final boolean DEBUG = false; |
| |
| private static final String HOTWORD_DETECTION_OP_MESSAGE = |
| "Providing hotword detection result to VoiceInteractionService"; |
| |
| // The error codes are used for onError callback |
| static final int HOTWORD_DETECTION_SERVICE_DIED = |
| HotwordDetectionServiceFailure.ERROR_CODE_BINDING_DIED; |
| static final int CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION = |
| HotwordDetectionServiceFailure.ERROR_CODE_ON_DETECTED_SECURITY_EXCEPTION; |
| static final int CALLBACK_DETECT_TIMEOUT = |
| HotwordDetectionServiceFailure.ERROR_CODE_DETECT_TIMEOUT; |
| static final int CALLBACK_ONDETECTED_STREAM_COPY_ERROR = |
| HotwordDetectionServiceFailure.ERROR_CODE_ON_DETECTED_STREAM_COPY_FAILURE; |
| static final int CALLBACK_COPY_AUDIO_DATA_FAILURE = |
| HotwordDetectionServiceFailure.ERROR_CODE_COPY_AUDIO_DATA_FAILURE; |
| |
| // TODO: These constants need to be refined. |
| private static final long MAX_UPDATE_TIMEOUT_MILLIS = 30000; |
| private static final long EXTERNAL_HOTWORD_CLEANUP_MILLIS = 2000; |
| private static final Duration MAX_UPDATE_TIMEOUT_DURATION = |
| Duration.ofMillis(MAX_UPDATE_TIMEOUT_MILLIS); |
| |
| // Hotword metrics |
| private static final int METRICS_INIT_UNKNOWN_TIMEOUT = |
| HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_TIMEOUT; |
| private static final int METRICS_INIT_UNKNOWN_NO_VALUE = |
| HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_NO_VALUE; |
| private static final int METRICS_INIT_UNKNOWN_OVER_MAX_CUSTOM_VALUE = |
| HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_UNKNOWN_OVER_MAX_CUSTOM_VALUE; |
| private static final int METRICS_INIT_CALLBACK_STATE_ERROR = |
| HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_ERROR; |
| private static final int METRICS_INIT_CALLBACK_STATE_SUCCESS = |
| HOTWORD_DETECTION_SERVICE_INIT_RESULT_REPORTED__RESULT__CALLBACK_INIT_STATE_SUCCESS; |
| |
| static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_SECURITY_EXCEPTION = |
| HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_SECURITY_EXCEPTION; |
| static final int METRICS_KEYPHRASE_TRIGGERED_DETECT_UNEXPECTED_CALLBACK = |
| HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__DETECT_UNEXPECTED_CALLBACK; |
| static final int METRICS_KEYPHRASE_TRIGGERED_REJECT_UNEXPECTED_CALLBACK = |
| HOTWORD_DETECTOR_KEYPHRASE_TRIGGERED__RESULT__REJECT_UNEXPECTED_CALLBACK; |
| |
| private static final int METRICS_EXTERNAL_SOURCE_DETECTED = |
| HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECTED; |
| private static final int METRICS_EXTERNAL_SOURCE_REJECTED = |
| HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_REJECTED; |
| private static final int EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION = |
| HOTWORD_DETECTOR_EVENTS__EVENT__EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION; |
| private static final int METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION = |
| HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_STATUS_REPORTED_EXCEPTION; |
| |
| private final Executor mAudioCopyExecutor = Executors.newCachedThreadPool(); |
| // TODO: This may need to be a Handler(looper) |
| final ScheduledExecutorService mScheduledExecutorService; |
| private final AppOpsManager mAppOpsManager; |
| final HotwordAudioStreamCopier mHotwordAudioStreamCopier; |
| final AtomicBoolean mUpdateStateAfterStartFinished = new AtomicBoolean(false); |
| final IHotwordRecognitionStatusCallback mCallback; |
| |
| final Object mLock; |
| final int mVoiceInteractionServiceUid; |
| final Context mContext; |
| |
| @Nullable AttentionManagerInternal mAttentionManagerInternal = null; |
| |
| final AttentionManagerInternal.ProximityUpdateCallbackInternal mProximityCallbackInternal = |
| this::setProximityValue; |
| |
| /** Identity used for attributing app ops when delivering data to the Interactor. */ |
| @Nullable |
| private final Identity mVoiceInteractorIdentity; |
| @GuardedBy("mLock") |
| ParcelFileDescriptor mCurrentAudioSink; |
| @GuardedBy("mLock") |
| @NonNull HotwordDetectionConnection.ServiceConnection mRemoteDetectionService; |
| boolean mDebugHotwordLogging = false; |
| @GuardedBy("mLock") |
| private double mProximityMeters = PROXIMITY_UNKNOWN; |
| @GuardedBy("mLock") |
| private boolean mInitialized = false; |
| @GuardedBy("mLock") |
| private boolean mDestroyed = false; |
| @GuardedBy("mLock") |
| boolean mPerformingExternalSourceHotwordDetection; |
| @NonNull final IBinder mToken; |
| |
| DetectorSession( |
| @NonNull HotwordDetectionConnection.ServiceConnection remoteDetectionService, |
| @NonNull Object lock, @NonNull Context context, @NonNull IBinder token, |
| @NonNull IHotwordRecognitionStatusCallback callback, int voiceInteractionServiceUid, |
| Identity voiceInteractorIdentity, |
| @NonNull ScheduledExecutorService scheduledExecutorService, boolean logging) { |
| mRemoteDetectionService = remoteDetectionService; |
| mLock = lock; |
| mContext = context; |
| mToken = token; |
| mCallback = callback; |
| mVoiceInteractionServiceUid = voiceInteractionServiceUid; |
| mVoiceInteractorIdentity = voiceInteractorIdentity; |
| mAppOpsManager = mContext.getSystemService(AppOpsManager.class); |
| if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { |
| mHotwordAudioStreamCopier = new HotwordAudioStreamCopier(mAppOpsManager, |
| getDetectorType(), |
| mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, |
| mVoiceInteractorIdentity.attributionTag); |
| } else { |
| mHotwordAudioStreamCopier = null; |
| } |
| |
| mScheduledExecutorService = scheduledExecutorService; |
| mDebugHotwordLogging = logging; |
| |
| if (ENABLE_PROXIMITY_RESULT) { |
| mAttentionManagerInternal = LocalServices.getService(AttentionManagerInternal.class); |
| if (mAttentionManagerInternal != null) { |
| mAttentionManagerInternal.onStartProximityUpdates(mProximityCallbackInternal); |
| } |
| } |
| } |
| |
| @SuppressWarnings("GuardedBy") |
| private void updateStateAfterProcessStartLocked(PersistableBundle options, |
| SharedMemory sharedMemory) { |
| if (DEBUG) { |
| Slog.d(TAG, "updateStateAfterProcessStartLocked"); |
| } |
| AndroidFuture<Void> voidFuture = mRemoteDetectionService.postAsync(service -> { |
| AndroidFuture<Void> future = new AndroidFuture<>(); |
| IRemoteCallback statusCallback = new IRemoteCallback.Stub() { |
| @Override |
| public void sendResult(Bundle bundle) throws RemoteException { |
| if (DEBUG) { |
| Slog.d(TAG, "updateState finish"); |
| } |
| future.complete(null); |
| if (mUpdateStateAfterStartFinished.getAndSet(true)) { |
| Slog.w(TAG, "call callback after timeout"); |
| if (getDetectorType() |
| != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { |
| HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), |
| HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_UPDATE_STATE_AFTER_TIMEOUT, |
| mVoiceInteractionServiceUid); |
| } |
| return; |
| } |
| Pair<Integer, Integer> statusResultPair = getInitStatusAndMetricsResult(bundle); |
| int status = statusResultPair.first; |
| int initResultMetricsResult = statusResultPair.second; |
| try { |
| mCallback.onStatusReported(status); |
| if (getDetectorType() |
| != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { |
| HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), |
| initResultMetricsResult, mVoiceInteractionServiceUid); |
| } |
| } catch (RemoteException e) { |
| Slog.w(TAG, "Failed to report initialization status: " + e); |
| if (getDetectorType() |
| != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { |
| HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), |
| METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, |
| mVoiceInteractionServiceUid); |
| } |
| } |
| } |
| }; |
| try { |
| service.updateState(options, sharedMemory, statusCallback); |
| if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { |
| HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), |
| HOTWORD_DETECTOR_EVENTS__EVENT__REQUEST_UPDATE_STATE, |
| mVoiceInteractionServiceUid); |
| } |
| } catch (RemoteException e) { |
| // TODO: (b/181842909) Report an error to voice interactor |
| Slog.w(TAG, "Failed to updateState for HotwordDetectionService", e); |
| if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { |
| HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), |
| HOTWORD_DETECTOR_EVENTS__EVENT__CALL_UPDATE_STATE_EXCEPTION, |
| mVoiceInteractionServiceUid); |
| } |
| } |
| return future.orTimeout(MAX_UPDATE_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); |
| }).whenComplete((res, err) -> { |
| if (err instanceof TimeoutException) { |
| Slog.w(TAG, "updateState timed out"); |
| if (mUpdateStateAfterStartFinished.getAndSet(true)) { |
| return; |
| } |
| try { |
| mCallback.onStatusReported(INITIALIZATION_STATUS_UNKNOWN); |
| if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { |
| HotwordMetricsLogger.writeServiceInitResultEvent(getDetectorType(), |
| METRICS_INIT_UNKNOWN_TIMEOUT, mVoiceInteractionServiceUid); |
| } |
| } catch (RemoteException e) { |
| Slog.w(TAG, "Failed to report initialization status UNKNOWN", e); |
| if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { |
| HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), |
| METRICS_CALLBACK_ON_STATUS_REPORTED_EXCEPTION, |
| mVoiceInteractionServiceUid); |
| } |
| } |
| } else if (err != null) { |
| Slog.w(TAG, "Failed to update state: " + err); |
| } |
| }); |
| if (voidFuture == null) { |
| Slog.w(TAG, "Failed to create AndroidFuture"); |
| } |
| } |
| |
| private static Pair<Integer, Integer> getInitStatusAndMetricsResult(Bundle bundle) { |
| if (bundle == null) { |
| return new Pair<>(INITIALIZATION_STATUS_UNKNOWN, METRICS_INIT_UNKNOWN_NO_VALUE); |
| } |
| int status = bundle.getInt(KEY_INITIALIZATION_STATUS, INITIALIZATION_STATUS_UNKNOWN); |
| if (status > HotwordDetectionService.getMaxCustomInitializationStatus()) { |
| return new Pair<>(INITIALIZATION_STATUS_UNKNOWN, |
| status == INITIALIZATION_STATUS_UNKNOWN |
| ? METRICS_INIT_UNKNOWN_NO_VALUE |
| : METRICS_INIT_UNKNOWN_OVER_MAX_CUSTOM_VALUE); |
| } |
| // TODO: should guard against negative here |
| int metricsResult = status == INITIALIZATION_STATUS_SUCCESS |
| ? METRICS_INIT_CALLBACK_STATE_SUCCESS |
| : METRICS_INIT_CALLBACK_STATE_ERROR; |
| return new Pair<>(status, metricsResult); |
| } |
| |
| @SuppressWarnings("GuardedBy") |
| void updateStateLocked(PersistableBundle options, SharedMemory sharedMemory, |
| Instant lastRestartInstant) { |
| if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { |
| HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), |
| HOTWORD_DETECTOR_EVENTS__EVENT__APP_REQUEST_UPDATE_STATE, |
| mVoiceInteractionServiceUid); |
| } |
| // Prevent doing the init late, so restart is handled equally to a clean process start. |
| // TODO(b/191742511): this logic needs a test |
| if (!mUpdateStateAfterStartFinished.get() && Instant.now().minus( |
| MAX_UPDATE_TIMEOUT_DURATION).isBefore(lastRestartInstant)) { |
| Slog.v(TAG, "call updateStateAfterProcessStartLocked"); |
| updateStateAfterProcessStartLocked(options, sharedMemory); |
| } else { |
| mRemoteDetectionService.run( |
| service -> service.updateState(options, sharedMemory, /* callback= */ null)); |
| } |
| } |
| |
| void startListeningFromExternalSourceLocked( |
| ParcelFileDescriptor audioStream, |
| AudioFormat audioFormat, |
| @Nullable PersistableBundle options, |
| IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { |
| if (DEBUG) { |
| Slog.d(TAG, "startListeningFromExternalSourceLocked"); |
| } |
| |
| handleExternalSourceHotwordDetectionLocked( |
| audioStream, |
| audioFormat, |
| options, |
| callback); |
| } |
| |
| @SuppressWarnings("GuardedBy") |
| private void handleExternalSourceHotwordDetectionLocked( |
| ParcelFileDescriptor audioStream, |
| AudioFormat audioFormat, |
| @Nullable PersistableBundle options, |
| IMicrophoneHotwordDetectionVoiceInteractionCallback callback) { |
| if (DEBUG) { |
| Slog.d(TAG, "#handleExternalSourceHotwordDetectionLocked"); |
| } |
| if (mPerformingExternalSourceHotwordDetection) { |
| Slog.i(TAG, "Hotword validation is already in progress for external source."); |
| return; |
| } |
| |
| InputStream audioSource = new ParcelFileDescriptor.AutoCloseInputStream(audioStream); |
| |
| Pair<ParcelFileDescriptor, ParcelFileDescriptor> clientPipe = createPipe(); |
| if (clientPipe == null) { |
| // TODO: Need to propagate as unknown error or something? |
| return; |
| } |
| ParcelFileDescriptor serviceAudioSink = clientPipe.second; |
| ParcelFileDescriptor serviceAudioSource = clientPipe.first; |
| |
| mCurrentAudioSink = serviceAudioSink; |
| mPerformingExternalSourceHotwordDetection = true; |
| |
| mAudioCopyExecutor.execute(() -> { |
| try (InputStream source = audioSource; |
| OutputStream fos = |
| new ParcelFileDescriptor.AutoCloseOutputStream(serviceAudioSink)) { |
| |
| byte[] buffer = new byte[1024]; |
| while (true) { |
| int bytesRead = source.read(buffer, 0, 1024); |
| |
| if (bytesRead < 0) { |
| Slog.i(TAG, "Reached end of stream for external hotword"); |
| break; |
| } |
| |
| // TODO: First write to ring buffer to make sure we don't lose data if the next |
| // statement fails. |
| // ringBuffer.append(buffer, bytesRead); |
| fos.write(buffer, 0, bytesRead); |
| } |
| } catch (IOException e) { |
| Slog.w(TAG, "Failed supplying audio data to validator", e); |
| |
| try { |
| callback.onError( |
| new HotwordDetectionServiceFailure(CALLBACK_COPY_AUDIO_DATA_FAILURE, |
| "Copy audio data failure for external source detection.")); |
| } catch (RemoteException ex) { |
| Slog.w(TAG, "Failed to report onError status: " + ex); |
| if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { |
| HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), |
| HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, |
| mVoiceInteractionServiceUid); |
| } |
| } |
| } finally { |
| synchronized (mLock) { |
| mPerformingExternalSourceHotwordDetection = false; |
| closeExternalAudioStreamLocked("start external source"); |
| } |
| } |
| }); |
| |
| // TODO: handle cancellations well |
| // TODO: what if we cancelled and started a new one? |
| mRemoteDetectionService.run( |
| service -> { |
| service.detectFromMicrophoneSource( |
| serviceAudioSource, |
| // TODO: consider making a proxy callback + copy of audio format |
| AUDIO_SOURCE_EXTERNAL, |
| audioFormat, |
| options, |
| new IDspHotwordDetectionCallback.Stub() { |
| @Override |
| public void onRejected(HotwordRejectedResult result) |
| throws RemoteException { |
| synchronized (mLock) { |
| mPerformingExternalSourceHotwordDetection = false; |
| HotwordMetricsLogger.writeDetectorEvent( |
| getDetectorType(), |
| METRICS_EXTERNAL_SOURCE_REJECTED, |
| mVoiceInteractionServiceUid); |
| mScheduledExecutorService.schedule( |
| () -> { |
| bestEffortClose(serviceAudioSink, audioSource); |
| }, |
| EXTERNAL_HOTWORD_CLEANUP_MILLIS, |
| TimeUnit.MILLISECONDS); |
| |
| callback.onRejected(result); |
| |
| if (result != null) { |
| Slog.i(TAG, "Egressed 'hotword rejected result' " |
| + "from hotword trusted process"); |
| if (mDebugHotwordLogging) { |
| Slog.i(TAG, "Egressed detected result: " + result); |
| } |
| } |
| } |
| } |
| |
| @Override |
| public void onDetected(HotwordDetectedResult triggerResult) |
| throws RemoteException { |
| synchronized (mLock) { |
| mPerformingExternalSourceHotwordDetection = false; |
| HotwordMetricsLogger.writeDetectorEvent( |
| getDetectorType(), |
| METRICS_EXTERNAL_SOURCE_DETECTED, |
| mVoiceInteractionServiceUid); |
| mScheduledExecutorService.schedule( |
| () -> { |
| bestEffortClose(serviceAudioSink, audioSource); |
| }, |
| EXTERNAL_HOTWORD_CLEANUP_MILLIS, |
| TimeUnit.MILLISECONDS); |
| |
| try { |
| enforcePermissionsForDataDelivery(); |
| } catch (SecurityException e) { |
| HotwordMetricsLogger.writeDetectorEvent( |
| getDetectorType(), |
| EXTERNAL_SOURCE_DETECT_SECURITY_EXCEPTION, |
| mVoiceInteractionServiceUid); |
| callback.onError(new HotwordDetectionServiceFailure( |
| CALLBACK_ONDETECTED_GOT_SECURITY_EXCEPTION, |
| "Security exception occurs in #onDetected" |
| + " method.")); |
| return; |
| } |
| HotwordDetectedResult newResult; |
| try { |
| newResult = mHotwordAudioStreamCopier |
| .startCopyingAudioStreams(triggerResult); |
| } catch (IOException e) { |
| // TODO: Write event |
| callback.onError(new HotwordDetectionServiceFailure( |
| CALLBACK_ONDETECTED_STREAM_COPY_ERROR, |
| "Copy audio stream failure.")); |
| return; |
| } |
| callback.onDetected(newResult, /* audioFormat= */ null, |
| /* audioStream= */ null); |
| Slog.i(TAG, "Egressed " |
| + HotwordDetectedResult.getUsageSize(newResult) |
| + " bits from hotword trusted process"); |
| if (mDebugHotwordLogging) { |
| Slog.i(TAG, |
| "Egressed detected result: " + newResult); |
| } |
| } |
| } |
| }); |
| |
| // A copy of this has been created and passed to the hotword validator |
| bestEffortClose(serviceAudioSource); |
| }); |
| HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), |
| HOTWORD_DETECTOR_EVENTS__EVENT__START_EXTERNAL_SOURCE_DETECTION, |
| mVoiceInteractionServiceUid); |
| } |
| |
| void initialize(@Nullable PersistableBundle options, @Nullable SharedMemory sharedMemory) { |
| synchronized (mLock) { |
| if (mInitialized || mDestroyed) { |
| return; |
| } |
| updateStateAfterProcessStartLocked(options, sharedMemory); |
| mInitialized = true; |
| } |
| } |
| |
| @SuppressWarnings("GuardedBy") |
| void destroyLocked() { |
| mDestroyed = true; |
| mDebugHotwordLogging = false; |
| mRemoteDetectionService = null; |
| if (mAttentionManagerInternal != null) { |
| mAttentionManagerInternal.onStopProximityUpdates(mProximityCallbackInternal); |
| } |
| } |
| |
| void setDebugHotwordLoggingLocked(boolean logging) { |
| Slog.v(TAG, "setDebugHotwordLoggingLocked: " + logging); |
| mDebugHotwordLogging = logging; |
| } |
| |
| @SuppressWarnings("GuardedBy") |
| void updateRemoteSandboxedDetectionServiceLocked( |
| @NonNull HotwordDetectionConnection.ServiceConnection remoteDetectionService) { |
| mRemoteDetectionService = remoteDetectionService; |
| } |
| |
| void reportErrorLocked(int errorCode, @NonNull String errorMessage) { |
| try { |
| // TODO: Use instanceof(this) to get different detector to set the right error source. |
| mCallback.onDetectionFailure( |
| new HotwordDetectionServiceFailure(errorCode, errorMessage)); |
| } catch (RemoteException e) { |
| Slog.w(TAG, "Failed to report onError status: " + e); |
| if (getDetectorType() != HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR) { |
| HotwordMetricsLogger.writeDetectorEvent(getDetectorType(), |
| HOTWORD_DETECTOR_EVENTS__EVENT__CALLBACK_ON_ERROR_EXCEPTION, |
| mVoiceInteractionServiceUid); |
| } |
| } |
| } |
| |
| /** |
| * Called when the trusted process is restarted. |
| */ |
| abstract void informRestartProcessLocked(); |
| |
| boolean isSameCallback(@Nullable IHotwordRecognitionStatusCallback callback) { |
| synchronized (mLock) { |
| if (callback == null) { |
| return false; |
| } |
| return mCallback.asBinder().equals(callback.asBinder()); |
| } |
| } |
| |
| boolean isSameToken(@NonNull IBinder token) { |
| synchronized (mLock) { |
| if (token == null) { |
| return false; |
| } |
| return mToken == token; |
| } |
| } |
| |
| boolean isDestroyed() { |
| synchronized (mLock) { |
| return mDestroyed; |
| } |
| } |
| |
| private static Pair<ParcelFileDescriptor, ParcelFileDescriptor> createPipe() { |
| ParcelFileDescriptor[] fileDescriptors; |
| try { |
| fileDescriptors = ParcelFileDescriptor.createPipe(); |
| } catch (IOException e) { |
| Slog.e(TAG, "Failed to create audio stream pipe", e); |
| return null; |
| } |
| |
| return Pair.create(fileDescriptors[0], fileDescriptors[1]); |
| } |
| |
| void saveProximityValueToBundle(HotwordDetectedResult result) { |
| synchronized (mLock) { |
| if (result != null && mProximityMeters != PROXIMITY_UNKNOWN) { |
| result.setProximity(mProximityMeters); |
| } |
| } |
| } |
| |
| private void setProximityValue(double proximityMeters) { |
| synchronized (mLock) { |
| mProximityMeters = proximityMeters; |
| } |
| } |
| |
| @SuppressWarnings("GuardedBy") |
| void closeExternalAudioStreamLocked(String reason) { |
| if (mCurrentAudioSink != null) { |
| Slog.i(TAG, "Closing external audio stream to hotword detector: " + reason); |
| bestEffortClose(mCurrentAudioSink); |
| mCurrentAudioSink = null; |
| } |
| } |
| |
| private static void bestEffortClose(Closeable... closeables) { |
| for (Closeable closeable : closeables) { |
| bestEffortClose(closeable); |
| } |
| } |
| |
| private static void bestEffortClose(Closeable closeable) { |
| try { |
| closeable.close(); |
| } catch (IOException e) { |
| if (DEBUG) { |
| Slog.w(TAG, "Failed closing", e); |
| } |
| } |
| } |
| |
| // TODO: Share this code with SoundTriggerMiddlewarePermission. |
| void enforcePermissionsForDataDelivery() { |
| Binder.withCleanCallingIdentity(() -> { |
| synchronized (mLock) { |
| enforcePermissionForPreflight(mContext, mVoiceInteractorIdentity, RECORD_AUDIO); |
| int hotwordOp = AppOpsManager.strOpToOp(AppOpsManager.OPSTR_RECORD_AUDIO_HOTWORD); |
| mAppOpsManager.noteOpNoThrow(hotwordOp, |
| mVoiceInteractorIdentity.uid, mVoiceInteractorIdentity.packageName, |
| mVoiceInteractorIdentity.attributionTag, HOTWORD_DETECTION_OP_MESSAGE); |
| enforcePermissionForDataDelivery(mContext, mVoiceInteractorIdentity, |
| CAPTURE_AUDIO_HOTWORD, HOTWORD_DETECTION_OP_MESSAGE); |
| } |
| }); |
| } |
| |
| /** |
| * Throws a {@link SecurityException} if the given identity has no permission to receive data. |
| * |
| * @param context A {@link Context}, used for permission checks. |
| * @param identity The identity to check. |
| * @param permission The identifier of the permission we want to check. |
| * @param reason The reason why we're requesting the permission, for auditing purposes. |
| */ |
| private static void enforcePermissionForDataDelivery(@NonNull Context context, |
| @NonNull Identity identity, @NonNull String permission, @NonNull String reason) { |
| final int status = PermissionUtil.checkPermissionForDataDelivery(context, identity, |
| permission, reason); |
| if (status != PermissionChecker.PERMISSION_GRANTED) { |
| throw new SecurityException( |
| TextUtils.formatSimple("Failed to obtain permission %s for identity %s", |
| permission, |
| SoundTriggerSessionPermissionsDecorator.toString(identity))); |
| } |
| } |
| |
| @RequiresPermission(allOf = {READ_COMPAT_CHANGE_CONFIG, LOG_COMPAT_CHANGE}) |
| void enforceExtraKeyphraseIdNotLeaked(HotwordDetectedResult result, |
| SoundTrigger.KeyphraseRecognitionEvent recognitionEvent) { |
| if (!CompatChanges.isChangeEnabled(ENFORCE_HOTWORD_PHRASE_ID, |
| mVoiceInteractionServiceUid)) { |
| return; |
| } |
| // verify the phrase ID in HotwordDetectedResult is not exposing extra phrases |
| // the DSP did not detect |
| for (SoundTrigger.KeyphraseRecognitionExtra keyphrase : recognitionEvent.keyphraseExtras) { |
| if (keyphrase.getKeyphraseId() == result.getHotwordPhraseId()) { |
| return; |
| } |
| } |
| throw new SecurityException("Ignoring #onDetected due to trusted service " |
| + "sharing a keyphrase ID which the DSP did not detect"); |
| } |
| |
| private int getDetectorType() { |
| if (this instanceof DspTrustedHotwordDetectorSession) { |
| return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_DSP; |
| } else if (this instanceof SoftwareTrustedHotwordDetectorSession) { |
| return HotwordDetector.DETECTOR_TYPE_TRUSTED_HOTWORD_SOFTWARE; |
| } else if (this instanceof VisualQueryDetectorSession) { |
| return HotwordDetector.DETECTOR_TYPE_VISUAL_QUERY_DETECTOR; |
| } |
| Slog.v(TAG, "Unexpected detector type"); |
| return -1; |
| } |
| |
| @SuppressWarnings("GuardedBy") |
| public void dumpLocked(String prefix, PrintWriter pw) { |
| pw.print(prefix); pw.print("mCallback="); pw.println(mCallback); |
| pw.print(prefix); pw.print("mUpdateStateAfterStartFinished="); |
| pw.println(mUpdateStateAfterStartFinished); |
| pw.print(prefix); pw.print("mInitialized="); pw.println(mInitialized); |
| pw.print(prefix); pw.print("mDestroyed="); pw.println(mDestroyed); |
| pw.print(prefix); pw.print("DetectorType="); |
| pw.println(HotwordDetector.detectorTypeToString(getDetectorType())); |
| pw.print(prefix); pw.print("mPerformingExternalSourceHotwordDetection="); |
| pw.println(mPerformingExternalSourceHotwordDetection); |
| } |
| } |
