| /** |
| * Copyright (C) 2014 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package com.android.server.voiceinteraction; |
| |
| import android.content.BroadcastReceiver; |
| import android.content.Context; |
| import android.content.Intent; |
| import android.content.IntentFilter; |
| import android.hardware.soundtrigger.IRecognitionStatusCallback; |
| import android.hardware.soundtrigger.SoundTrigger; |
| import android.hardware.soundtrigger.SoundTrigger.Keyphrase; |
| import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionEvent; |
| import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra; |
| import android.hardware.soundtrigger.SoundTrigger.KeyphraseSoundModel; |
| import android.hardware.soundtrigger.SoundTrigger.ModuleProperties; |
| import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig; |
| import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent; |
| import android.hardware.soundtrigger.SoundTrigger.SoundModelEvent; |
| import android.hardware.soundtrigger.SoundTriggerModule; |
| import android.os.PowerManager; |
| import android.os.RemoteException; |
| import android.telephony.PhoneStateListener; |
| import android.telephony.TelephonyManager; |
| import android.util.Slog; |
| |
| import java.io.FileDescriptor; |
| import java.io.PrintWriter; |
| import java.util.ArrayList; |
| |
| /** |
| * Helper for {@link SoundTrigger} APIs. |
| * Currently this just acts as an abstraction over all SoundTrigger API calls. |
| * |
| * @hide |
| */ |
| public class SoundTriggerHelper implements SoundTrigger.StatusListener { |
| static final String TAG = "SoundTriggerHelper"; |
| static final boolean DBG = false; |
| |
| /** |
| * Return codes for {@link #startRecognition(int, KeyphraseSoundModel, |
| * IRecognitionStatusCallback, RecognitionConfig)}, |
| * {@link #stopRecognition(int, IRecognitionStatusCallback)} |
| */ |
| public static final int STATUS_ERROR = SoundTrigger.STATUS_ERROR; |
| public static final int STATUS_OK = SoundTrigger.STATUS_OK; |
| |
| private static final int INVALID_VALUE = Integer.MIN_VALUE; |
| |
| /** The {@link ModuleProperties} for the system, or null if none exists. */ |
| final ModuleProperties moduleProperties; |
| |
| /** The properties for the DSP module */ |
| private SoundTriggerModule mModule; |
| private final Object mLock = new Object(); |
| private final Context mContext; |
| private final TelephonyManager mTelephonyManager; |
| private final PhoneStateListener mPhoneStateListener; |
| private final PowerManager mPowerManager; |
| |
| // TODO: Since many layers currently only deal with one recognition |
| // we simplify things by assuming one listener here too. |
| private IRecognitionStatusCallback mActiveListener; |
| private int mKeyphraseId = INVALID_VALUE; |
| private int mCurrentSoundModelHandle = INVALID_VALUE; |
| private KeyphraseSoundModel mCurrentSoundModel = null; |
| // FIXME: Ideally this should not be stored if allowMultipleTriggers happens at a lower layer. |
| private RecognitionConfig mRecognitionConfig = null; |
| private boolean mRequested = false; |
| private boolean mCallActive = false; |
| private boolean mIsPowerSaveMode = false; |
| // Indicates if the native sound trigger service is disabled or not. |
| // This is an indirect indication of the microphone being open in some other application. |
| private boolean mServiceDisabled = false; |
| private boolean mStarted = false; |
| private boolean mRecognitionAborted = false; |
| private PowerSaveModeListener mPowerSaveModeListener; |
| |
| SoundTriggerHelper(Context context) { |
| ArrayList <ModuleProperties> modules = new ArrayList<>(); |
| int status = SoundTrigger.listModules(modules); |
| mContext = context; |
| mTelephonyManager = (TelephonyManager) context.getSystemService(Context.TELEPHONY_SERVICE); |
| mPowerManager = (PowerManager) context.getSystemService(Context.POWER_SERVICE); |
| mPhoneStateListener = new MyCallStateListener(); |
| if (status != SoundTrigger.STATUS_OK || modules.size() == 0) { |
| Slog.w(TAG, "listModules status=" + status + ", # of modules=" + modules.size()); |
| moduleProperties = null; |
| mModule = null; |
| } else { |
| // TODO: Figure out how to determine which module corresponds to the DSP hardware. |
| moduleProperties = modules.get(0); |
| } |
| } |
| |
| /** |
| * Starts recognition for the given keyphraseId. |
| * |
| * @param keyphraseId The identifier of the keyphrase for which |
| * the recognition is to be started. |
| * @param soundModel The sound model to use for recognition. |
| * @param listener The listener for the recognition events related to the given keyphrase. |
| * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. |
| */ |
| int startRecognition(int keyphraseId, |
| KeyphraseSoundModel soundModel, |
| IRecognitionStatusCallback listener, |
| RecognitionConfig recognitionConfig) { |
| if (soundModel == null || listener == null || recognitionConfig == null) { |
| return STATUS_ERROR; |
| } |
| |
| synchronized (mLock) { |
| if (DBG) { |
| Slog.d(TAG, "startRecognition for keyphraseId=" + keyphraseId |
| + " soundModel=" + soundModel + ", listener=" + listener.asBinder() |
| + ", recognitionConfig=" + recognitionConfig); |
| Slog.d(TAG, "moduleProperties=" + moduleProperties); |
| Slog.d(TAG, "current listener=" |
| + (mActiveListener == null ? "null" : mActiveListener.asBinder())); |
| Slog.d(TAG, "current SoundModel handle=" + mCurrentSoundModelHandle); |
| Slog.d(TAG, "current SoundModel UUID=" |
| + (mCurrentSoundModel == null ? null : mCurrentSoundModel.uuid)); |
| } |
| |
| if (!mStarted) { |
| // Get the current call state synchronously for the first recognition. |
| mCallActive = mTelephonyManager.getCallState() != TelephonyManager.CALL_STATE_IDLE; |
| // Register for call state changes when the first call to start recognition occurs. |
| mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_CALL_STATE); |
| |
| // Register for power saver mode changes when the first call to start recognition |
| // occurs. |
| if (mPowerSaveModeListener == null) { |
| mPowerSaveModeListener = new PowerSaveModeListener(); |
| mContext.registerReceiver(mPowerSaveModeListener, |
| new IntentFilter(PowerManager.ACTION_POWER_SAVE_MODE_CHANGED)); |
| } |
| mIsPowerSaveMode = mPowerManager.isPowerSaveMode(); |
| } |
| |
| if (moduleProperties == null) { |
| Slog.w(TAG, "Attempting startRecognition without the capability"); |
| return STATUS_ERROR; |
| } |
| if (mModule == null) { |
| mModule = SoundTrigger.attachModule(moduleProperties.id, this, null); |
| if (mModule == null) { |
| Slog.w(TAG, "startRecognition cannot attach to sound trigger module"); |
| return STATUS_ERROR; |
| } |
| } |
| |
| // Unload the previous model if the current one isn't invalid |
| // and, it's not the same as the new one. |
| // This helps use cache and reuse the model and just start/stop it when necessary. |
| if (mCurrentSoundModelHandle != INVALID_VALUE |
| && !soundModel.equals(mCurrentSoundModel)) { |
| Slog.w(TAG, "Unloading previous sound model"); |
| int status = mModule.unloadSoundModel(mCurrentSoundModelHandle); |
| if (status != SoundTrigger.STATUS_OK) { |
| Slog.w(TAG, "unloadSoundModel call failed with " + status); |
| } |
| internalClearSoundModelLocked(); |
| mStarted = false; |
| } |
| |
| // If the previous recognition was by a different listener, |
| // Notify them that it was stopped. |
| if (mActiveListener != null && mActiveListener.asBinder() != listener.asBinder()) { |
| Slog.w(TAG, "Canceling previous recognition"); |
| try { |
| mActiveListener.onError(STATUS_ERROR); |
| } catch (RemoteException e) { |
| Slog.w(TAG, "RemoteException in onDetectionStopped", e); |
| } |
| mActiveListener = null; |
| } |
| |
| // Load the sound model if the current one is null. |
| int soundModelHandle = mCurrentSoundModelHandle; |
| if (mCurrentSoundModelHandle == INVALID_VALUE |
| || mCurrentSoundModel == null) { |
| int[] handle = new int[] { INVALID_VALUE }; |
| int status = mModule.loadSoundModel(soundModel, handle); |
| if (status != SoundTrigger.STATUS_OK) { |
| Slog.w(TAG, "loadSoundModel call failed with " + status); |
| return status; |
| } |
| if (handle[0] == INVALID_VALUE) { |
| Slog.w(TAG, "loadSoundModel call returned invalid sound model handle"); |
| return STATUS_ERROR; |
| } |
| soundModelHandle = handle[0]; |
| } else { |
| if (DBG) Slog.d(TAG, "Reusing previously loaded sound model"); |
| } |
| |
| // Start the recognition. |
| mRequested = true; |
| mKeyphraseId = keyphraseId; |
| mCurrentSoundModelHandle = soundModelHandle; |
| mCurrentSoundModel = soundModel; |
| mRecognitionConfig = recognitionConfig; |
| // Register the new listener. This replaces the old one. |
| // There can only be a maximum of one active listener at any given time. |
| mActiveListener = listener; |
| |
| return updateRecognitionLocked(false /* don't notify for synchronous calls */); |
| } |
| } |
| |
| /** |
| * Stops recognition for the given {@link Keyphrase} if a recognition is |
| * currently active. |
| * |
| * @param keyphraseId The identifier of the keyphrase for which |
| * the recognition is to be stopped. |
| * @param listener The listener for the recognition events related to the given keyphrase. |
| * |
| * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}. |
| */ |
| int stopRecognition(int keyphraseId, IRecognitionStatusCallback listener) { |
| if (listener == null) { |
| return STATUS_ERROR; |
| } |
| |
| synchronized (mLock) { |
| if (DBG) { |
| Slog.d(TAG, "stopRecognition for keyphraseId=" + keyphraseId |
| + ", listener=" + listener.asBinder()); |
| Slog.d(TAG, "current listener=" |
| + (mActiveListener == null ? "null" : mActiveListener.asBinder())); |
| } |
| |
| if (moduleProperties == null || mModule == null) { |
| Slog.w(TAG, "Attempting stopRecognition without the capability"); |
| return STATUS_ERROR; |
| } |
| |
| if (mActiveListener == null) { |
| // startRecognition hasn't been called or it failed. |
| Slog.w(TAG, "Attempting stopRecognition without a successful startRecognition"); |
| return STATUS_ERROR; |
| } |
| if (mActiveListener.asBinder() != listener.asBinder()) { |
| // We don't allow a different listener to stop the recognition than the one |
| // that started it. |
| Slog.w(TAG, "Attempting stopRecognition for another recognition"); |
| return STATUS_ERROR; |
| } |
| |
| // Stop recognition if it's the current one, ignore otherwise. |
| mRequested = false; |
| int status = updateRecognitionLocked(false /* don't notify for synchronous calls */); |
| if (status != SoundTrigger.STATUS_OK) { |
| return status; |
| } |
| |
| // We leave the sound model loaded but not started, this helps us when we start |
| // back. |
| // Also clear the internal state once the recognition has been stopped. |
| internalClearStateLocked(); |
| return status; |
| } |
| } |
| |
| /** |
| * Stops all recognitions active currently and clears the internal state. |
| */ |
| void stopAllRecognitions() { |
| synchronized (mLock) { |
| if (moduleProperties == null || mModule == null) { |
| return; |
| } |
| |
| if (mCurrentSoundModelHandle == INVALID_VALUE) { |
| return; |
| } |
| |
| mRequested = false; |
| int status = updateRecognitionLocked(false /* don't notify for synchronous calls */); |
| internalClearStateLocked(); |
| } |
| } |
| |
| //---- SoundTrigger.StatusListener methods |
| @Override |
| public void onRecognition(RecognitionEvent event) { |
| if (event == null || !(event instanceof KeyphraseRecognitionEvent)) { |
| Slog.w(TAG, "Invalid recognition event!"); |
| return; |
| } |
| |
| if (DBG) Slog.d(TAG, "onRecognition: " + event); |
| synchronized (mLock) { |
| if (mActiveListener == null) { |
| Slog.w(TAG, "received onRecognition event without any listener for it"); |
| return; |
| } |
| switch (event.status) { |
| // Fire aborts/failures to all listeners since it's not tied to a keyphrase. |
| case SoundTrigger.RECOGNITION_STATUS_ABORT: |
| onRecognitionAbortLocked(); |
| break; |
| case SoundTrigger.RECOGNITION_STATUS_FAILURE: |
| onRecognitionFailureLocked(); |
| break; |
| case SoundTrigger.RECOGNITION_STATUS_SUCCESS: |
| onRecognitionSuccessLocked((KeyphraseRecognitionEvent) event); |
| break; |
| } |
| } |
| } |
| |
| @Override |
| public void onSoundModelUpdate(SoundModelEvent event) { |
| if (event == null) { |
| Slog.w(TAG, "Invalid sound model event!"); |
| return; |
| } |
| if (DBG) Slog.d(TAG, "onSoundModelUpdate: " + event); |
| synchronized (mLock) { |
| onSoundModelUpdatedLocked(event); |
| } |
| } |
| |
| @Override |
| public void onServiceStateChange(int state) { |
| if (DBG) Slog.d(TAG, "onServiceStateChange, state: " + state); |
| synchronized (mLock) { |
| onServiceStateChangedLocked(SoundTrigger.SERVICE_STATE_DISABLED == state); |
| } |
| } |
| |
| @Override |
| public void onServiceDied() { |
| Slog.e(TAG, "onServiceDied!!"); |
| synchronized (mLock) { |
| onServiceDiedLocked(); |
| } |
| } |
| |
| private void onCallStateChangedLocked(boolean callActive) { |
| if (mCallActive == callActive) { |
| // We consider multiple call states as being active |
| // so we check if something really changed or not here. |
| return; |
| } |
| mCallActive = callActive; |
| updateRecognitionLocked(true /* notify */); |
| } |
| |
| private void onPowerSaveModeChangedLocked(boolean isPowerSaveMode) { |
| if (mIsPowerSaveMode == isPowerSaveMode) { |
| return; |
| } |
| mIsPowerSaveMode = isPowerSaveMode; |
| updateRecognitionLocked(true /* notify */); |
| } |
| |
| private void onSoundModelUpdatedLocked(SoundModelEvent event) { |
| // TODO: Handle sound model update here. |
| } |
| |
| private void onServiceStateChangedLocked(boolean disabled) { |
| if (disabled == mServiceDisabled) { |
| return; |
| } |
| mServiceDisabled = disabled; |
| updateRecognitionLocked(true /* notify */); |
| } |
| |
| private void onRecognitionAbortLocked() { |
| Slog.w(TAG, "Recognition aborted"); |
| // If abort has been called, the hardware has already stopped recognition, so we shouldn't |
| // call it again when we process the state change. |
| mRecognitionAborted = true; |
| } |
| |
| private void onRecognitionFailureLocked() { |
| Slog.w(TAG, "Recognition failure"); |
| try { |
| if (mActiveListener != null) { |
| mActiveListener.onError(STATUS_ERROR); |
| } |
| } catch (RemoteException e) { |
| Slog.w(TAG, "RemoteException in onError", e); |
| } finally { |
| internalClearStateLocked(); |
| } |
| } |
| |
| private void onRecognitionSuccessLocked(KeyphraseRecognitionEvent event) { |
| Slog.i(TAG, "Recognition success"); |
| KeyphraseRecognitionExtra[] keyphraseExtras = |
| ((KeyphraseRecognitionEvent) event).keyphraseExtras; |
| if (keyphraseExtras == null || keyphraseExtras.length == 0) { |
| Slog.w(TAG, "Invalid keyphrase recognition event!"); |
| return; |
| } |
| // TODO: Handle more than one keyphrase extras. |
| if (mKeyphraseId != keyphraseExtras[0].id) { |
| Slog.w(TAG, "received onRecognition event for a different keyphrase"); |
| return; |
| } |
| |
| try { |
| if (mActiveListener != null) { |
| mActiveListener.onDetected((KeyphraseRecognitionEvent) event); |
| } |
| } catch (RemoteException e) { |
| Slog.w(TAG, "RemoteException in onDetected", e); |
| } |
| |
| mStarted = false; |
| mRequested = mRecognitionConfig.allowMultipleTriggers; |
| // TODO: Remove this block if the lower layer supports multiple triggers. |
| if (mRequested) { |
| updateRecognitionLocked(true /* notify */); |
| } |
| } |
| |
| private void onServiceDiedLocked() { |
| try { |
| if (mActiveListener != null) { |
| mActiveListener.onError(SoundTrigger.STATUS_DEAD_OBJECT); |
| } |
| } catch (RemoteException e) { |
| Slog.w(TAG, "RemoteException in onError", e); |
| } finally { |
| internalClearSoundModelLocked(); |
| internalClearStateLocked(); |
| if (mModule != null) { |
| mModule.detach(); |
| mModule = null; |
| } |
| } |
| } |
| |
| private int updateRecognitionLocked(boolean notify) { |
| if (mModule == null || moduleProperties == null |
| || mCurrentSoundModelHandle == INVALID_VALUE || mActiveListener == null) { |
| // Nothing to do here. |
| return STATUS_OK; |
| } |
| |
| boolean start = mRequested && !mCallActive && !mServiceDisabled && !mIsPowerSaveMode; |
| if (start == mStarted) { |
| // No-op. |
| return STATUS_OK; |
| } |
| |
| // See if the recognition needs to be started. |
| if (start) { |
| // Start recognition. |
| int status = mModule.startRecognition(mCurrentSoundModelHandle, mRecognitionConfig); |
| if (status != SoundTrigger.STATUS_OK) { |
| Slog.w(TAG, "startRecognition failed with " + status); |
| // Notify of error if needed. |
| if (notify) { |
| try { |
| mActiveListener.onError(status); |
| } catch (RemoteException e) { |
| Slog.w(TAG, "RemoteException in onError", e); |
| } |
| } |
| } else { |
| mStarted = true; |
| // Notify of resume if needed. |
| if (notify) { |
| try { |
| mActiveListener.onRecognitionResumed(); |
| } catch (RemoteException e) { |
| Slog.w(TAG, "RemoteException in onRecognitionResumed", e); |
| } |
| } |
| } |
| return status; |
| } else { |
| // Stop recognition (only if we haven't been aborted). |
| int status = STATUS_OK; |
| if (!mRecognitionAborted) { |
| status = mModule.stopRecognition(mCurrentSoundModelHandle); |
| } else { |
| mRecognitionAborted = false; |
| } |
| if (status != SoundTrigger.STATUS_OK) { |
| Slog.w(TAG, "stopRecognition call failed with " + status); |
| if (notify) { |
| try { |
| mActiveListener.onError(status); |
| } catch (RemoteException e) { |
| Slog.w(TAG, "RemoteException in onError", e); |
| } |
| } |
| } else { |
| mStarted = false; |
| // Notify of pause if needed. |
| if (notify) { |
| try { |
| mActiveListener.onRecognitionPaused(); |
| } catch (RemoteException e) { |
| Slog.w(TAG, "RemoteException in onRecognitionPaused", e); |
| } |
| } |
| } |
| return status; |
| } |
| } |
| |
| private void internalClearStateLocked() { |
| mStarted = false; |
| mRequested = false; |
| |
| mKeyphraseId = INVALID_VALUE; |
| mRecognitionConfig = null; |
| mActiveListener = null; |
| |
| // Unregister from call state changes. |
| mTelephonyManager.listen(mPhoneStateListener, PhoneStateListener.LISTEN_NONE); |
| |
| // Unregister from power save mode changes. |
| if (mPowerSaveModeListener != null) { |
| mContext.unregisterReceiver(mPowerSaveModeListener); |
| mPowerSaveModeListener = null; |
| } |
| } |
| |
| private void internalClearSoundModelLocked() { |
| mCurrentSoundModelHandle = INVALID_VALUE; |
| mCurrentSoundModel = null; |
| } |
| |
| class MyCallStateListener extends PhoneStateListener { |
| @Override |
| public void onCallStateChanged(int state, String arg1) { |
| if (DBG) Slog.d(TAG, "onCallStateChanged: " + state); |
| synchronized (mLock) { |
| onCallStateChangedLocked(TelephonyManager.CALL_STATE_IDLE != state); |
| } |
| } |
| } |
| |
| class PowerSaveModeListener extends BroadcastReceiver { |
| @Override |
| public void onReceive(Context context, Intent intent) { |
| if (!PowerManager.ACTION_POWER_SAVE_MODE_CHANGED.equals(intent.getAction())) { |
| return; |
| } |
| boolean active = mPowerManager.isPowerSaveMode(); |
| if (DBG) Slog.d(TAG, "onPowerSaveModeChanged: " + active); |
| synchronized (mLock) { |
| onPowerSaveModeChangedLocked(active); |
| } |
| } |
| } |
| |
| void dump(FileDescriptor fd, PrintWriter pw, String[] args) { |
| synchronized (mLock) { |
| pw.print(" module properties="); |
| pw.println(moduleProperties == null ? "null" : moduleProperties); |
| pw.print(" keyphrase ID="); pw.println(mKeyphraseId); |
| pw.print(" sound model handle="); pw.println(mCurrentSoundModelHandle); |
| pw.print(" sound model UUID="); |
| pw.println(mCurrentSoundModel == null ? "null" : mCurrentSoundModel.uuid); |
| pw.print(" current listener="); |
| pw.println(mActiveListener == null ? "null" : mActiveListener.asBinder()); |
| |
| pw.print(" requested="); pw.println(mRequested); |
| pw.print(" started="); pw.println(mStarted); |
| pw.print(" call active="); pw.println(mCallActive); |
| pw.print(" power save mode active="); pw.println(mIsPowerSaveMode); |
| pw.print(" service disabled="); pw.println(mServiceDisabled); |
| } |
| } |
| } |
