481 lines
17 KiB
Java
481 lines
17 KiB
Java
|
/**
|
||
|
|
* Copyright (C) 2014 The Android Open Source Project
|
||
|
|
*
|
||
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
* you may not use this file except in compliance with the License.
|
||
|
|
* You may obtain a copy of the License at
|
||
|
|
*
|
||
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
*
|
||
|
|
* Unless required by applicable law or agreed to in writing, software
|
||
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
* See the License for the specific language governing permissions and
|
||
|
|
* limitations under the License.
|
||
|
|
*/
|
||
|
|
|
||
|
|
package android.media.soundtrigger;
|
||
|
|
import static android.hardware.soundtrigger.SoundTrigger.STATUS_OK;
|
||
|
|
|
||
|
|
import android.annotation.IntDef;
|
||
|
|
import android.annotation.NonNull;
|
||
|
|
import android.annotation.Nullable;
|
||
|
|
import android.annotation.RequiresPermission;
|
||
|
|
import android.annotation.SystemApi;
|
||
|
|
import android.compat.annotation.UnsupportedAppUsage;
|
||
|
|
import android.hardware.soundtrigger.IRecognitionStatusCallback;
|
||
|
|
import android.hardware.soundtrigger.SoundTrigger;
|
||
|
|
import android.hardware.soundtrigger.SoundTrigger.GenericSoundModel;
|
||
|
|
import android.hardware.soundtrigger.SoundTrigger.ModuleProperties;
|
||
|
|
import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig;
|
||
|
|
import android.media.AudioFormat;
|
||
|
|
import android.os.Build;
|
||
|
|
import android.os.Handler;
|
||
|
|
import android.os.Looper;
|
||
|
|
import android.os.Message;
|
||
|
|
import android.os.ParcelUuid;
|
||
|
|
import android.os.RemoteException;
|
||
|
|
import android.util.Slog;
|
||
|
|
|
||
|
|
import com.android.internal.app.ISoundTriggerSession;
|
||
|
|
|
||
|
|
import java.io.PrintWriter;
|
||
|
|
import java.lang.annotation.Retention;
|
||
|
|
import java.lang.annotation.RetentionPolicy;
|
||
|
|
import java.util.UUID;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* A class that allows interaction with the actual sound trigger detection on the system.
|
||
|
|
* Sound trigger detection refers to a detectors that match generic sound patterns that are
|
||
|
|
* not voice-based. The voice-based recognition models should utilize the {@link
|
||
|
|
* VoiceInteractionService} instead. Access to this class is protected by a permission
|
||
|
|
* granted only to system or privileged apps.
|
||
|
|
* @deprecated use {@link SoundTriggerManager} directly
|
||
|
|
*
|
||
|
|
* @hide
|
||
|
|
*/
|
||
|
|
@Deprecated
|
||
|
|
@SystemApi
|
||
|
|
public final class SoundTriggerDetector {
|
||
|
|
private static final boolean DBG = false;
|
||
|
|
private static final String TAG = "SoundTriggerDetector";
|
||
|
|
|
||
|
|
private static final int MSG_AVAILABILITY_CHANGED = 1;
|
||
|
|
private static final int MSG_SOUND_TRIGGER_DETECTED = 2;
|
||
|
|
private static final int MSG_DETECTION_ERROR = 3;
|
||
|
|
private static final int MSG_DETECTION_PAUSE = 4;
|
||
|
|
private static final int MSG_DETECTION_RESUME = 5;
|
||
|
|
|
||
|
|
private final Object mLock = new Object();
|
||
|
|
|
||
|
|
private final ISoundTriggerSession mSoundTriggerSession;
|
||
|
|
private final GenericSoundModel mSoundModel;
|
||
|
|
private final Callback mCallback;
|
||
|
|
private final Handler mHandler;
|
||
|
|
private final RecognitionCallback mRecognitionCallback;
|
||
|
|
|
||
|
|
/** @hide */
|
||
|
|
@Retention(RetentionPolicy.SOURCE)
|
||
|
|
@IntDef(flag = true,
|
||
|
|
value = {
|
||
|
|
RECOGNITION_FLAG_NONE,
|
||
|
|
RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO,
|
||
|
|
RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS,
|
||
|
|
RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION,
|
||
|
|
RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION,
|
||
|
|
RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER,
|
||
|
|
})
|
||
|
|
public @interface RecognitionFlags {}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Empty flag for {@link #startRecognition(int)}.
|
||
|
|
*
|
||
|
|
* @hide
|
||
|
|
*/
|
||
|
|
public static final int RECOGNITION_FLAG_NONE = 0;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Recognition flag for {@link #startRecognition(int)} that indicates
|
||
|
|
* whether the trigger audio for hotword needs to be captured.
|
||
|
|
*/
|
||
|
|
public static final int RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO = 0x1;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Recognition flag for {@link #startRecognition(int)} that indicates
|
||
|
|
* whether the recognition should keep going on even after the
|
||
|
|
* model triggers.
|
||
|
|
* If this flag is specified, it's possible to get multiple
|
||
|
|
* triggers after a call to {@link #startRecognition(int)}, if the model
|
||
|
|
* triggers multiple times.
|
||
|
|
* When this isn't specified, the default behavior is to stop recognition once the
|
||
|
|
* trigger happens, till the caller starts recognition again.
|
||
|
|
*/
|
||
|
|
public static final int RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS = 0x2;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Audio capabilities flag for {@link #startRecognition(int)} that indicates
|
||
|
|
* if the underlying recognition should use AEC.
|
||
|
|
* This capability may or may not be supported by the system, and support can be queried
|
||
|
|
* by calling {@link SoundTriggerManager#getModuleProperties()} and checking
|
||
|
|
* {@link ModuleProperties#audioCapabilities}. The corresponding capabilities field for
|
||
|
|
* this flag is {@link SoundTrigger.ModuleProperties#AUDIO_CAPABILITY_ECHO_CANCELLATION}.
|
||
|
|
* If this flag is passed without the audio capability supported, there will be no audio effect
|
||
|
|
* applied.
|
||
|
|
*/
|
||
|
|
public static final int RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION = 0x4;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Audio capabilities flag for {@link #startRecognition(int)} that indicates
|
||
|
|
* if the underlying recognition should use noise suppression.
|
||
|
|
* This capability may or may not be supported by the system, and support can be queried
|
||
|
|
* by calling {@link SoundTriggerManager#getModuleProperties()} and checking
|
||
|
|
* {@link ModuleProperties#audioCapabilities}. The corresponding capabilities field for
|
||
|
|
* this flag is {@link SoundTrigger.ModuleProperties#AUDIO_CAPABILITY_NOISE_SUPPRESSION}.
|
||
|
|
* If this flag is passed without the audio capability supported, there will be no audio effect
|
||
|
|
* applied.
|
||
|
|
*/
|
||
|
|
public static final int RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION = 0x8;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Recognition flag for {@link #startRecognition(int)} that indicates whether the recognition
|
||
|
|
* should continue after battery saver mode is enabled.
|
||
|
|
* When this flag is specified, the caller will be checked for
|
||
|
|
* {@link android.Manifest.permission#SOUND_TRIGGER_RUN_IN_BATTERY_SAVER} permission granted.
|
||
|
|
*/
|
||
|
|
public static final int RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER = 0x10;
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Additional payload for {@link Callback#onDetected}.
|
||
|
|
*/
|
||
|
|
public static class EventPayload {
|
||
|
|
private final boolean mTriggerAvailable;
|
||
|
|
|
||
|
|
// Indicates if {@code captureSession} can be used to continue capturing more audio
|
||
|
|
// from the DSP hardware.
|
||
|
|
private final boolean mCaptureAvailable;
|
||
|
|
// The session to use when attempting to capture more audio from the DSP hardware.
|
||
|
|
private final int mCaptureSession;
|
||
|
|
private final AudioFormat mAudioFormat;
|
||
|
|
// Raw data associated with the event.
|
||
|
|
// This is the audio that triggered the keyphrase if {@code isTriggerAudio} is true.
|
||
|
|
private final byte[] mData;
|
||
|
|
|
||
|
|
private EventPayload(boolean triggerAvailable, boolean captureAvailable,
|
||
|
|
AudioFormat audioFormat, int captureSession, byte[] data) {
|
||
|
|
mTriggerAvailable = triggerAvailable;
|
||
|
|
mCaptureAvailable = captureAvailable;
|
||
|
|
mCaptureSession = captureSession;
|
||
|
|
mAudioFormat = audioFormat;
|
||
|
|
mData = data;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Gets the format of the audio obtained using {@link #getTriggerAudio()}.
|
||
|
|
* May be null if there's no audio present.
|
||
|
|
*/
|
||
|
|
@Nullable
|
||
|
|
public AudioFormat getCaptureAudioFormat() {
|
||
|
|
return mAudioFormat;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Gets the raw audio that triggered the detector.
|
||
|
|
* This may be null if the trigger audio isn't available.
|
||
|
|
* If non-null, the format of the audio can be obtained by calling
|
||
|
|
* {@link #getCaptureAudioFormat()}.
|
||
|
|
*
|
||
|
|
* @see AlwaysOnHotwordDetector#RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO
|
||
|
|
*/
|
||
|
|
@Nullable
|
||
|
|
public byte[] getTriggerAudio() {
|
||
|
|
if (mTriggerAvailable) {
|
||
|
|
return mData;
|
||
|
|
} else {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Gets the opaque data passed from the detection engine for the event.
|
||
|
|
* This may be null if it was not populated by the engine, or if the data is known to
|
||
|
|
* contain the trigger audio.
|
||
|
|
*
|
||
|
|
* @see #getTriggerAudio
|
||
|
|
*
|
||
|
|
* @hide
|
||
|
|
*/
|
||
|
|
@Nullable
|
||
|
|
@UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553)
|
||
|
|
public byte[] getData() {
|
||
|
|
if (!mTriggerAvailable) {
|
||
|
|
return mData;
|
||
|
|
} else {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Gets the session ID to start a capture from the DSP.
|
||
|
|
* This may be null if streaming capture isn't possible.
|
||
|
|
* If non-null, the format of the audio that can be captured can be
|
||
|
|
* obtained using {@link #getCaptureAudioFormat()}.
|
||
|
|
*
|
||
|
|
* TODO: Candidate for Public API when the API to start capture with a session ID
|
||
|
|
* is made public.
|
||
|
|
*
|
||
|
|
* TODO: Add this to {@link #getCaptureAudioFormat()}:
|
||
|
|
* "Gets the format of the audio obtained using {@link #getTriggerAudio()}
|
||
|
|
* or {@link #getCaptureSession()}. May be null if no audio can be obtained
|
||
|
|
* for either the trigger or a streaming session."
|
||
|
|
*
|
||
|
|
* TODO: Should this return a known invalid value instead?
|
||
|
|
*
|
||
|
|
* @hide
|
||
|
|
*/
|
||
|
|
@Nullable
|
||
|
|
@UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.R, trackingBug = 170729553)
|
||
|
|
public Integer getCaptureSession() {
|
||
|
|
if (mCaptureAvailable) {
|
||
|
|
return mCaptureSession;
|
||
|
|
} else {
|
||
|
|
return null;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
public static abstract class Callback {
|
||
|
|
/**
|
||
|
|
* Called when the availability of the sound model changes.
|
||
|
|
*/
|
||
|
|
public abstract void onAvailabilityChanged(int status);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Called when the sound model has triggered (such as when it matched a
|
||
|
|
* given sound pattern).
|
||
|
|
*/
|
||
|
|
public abstract void onDetected(@NonNull EventPayload eventPayload);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Called when the detection fails due to an error.
|
||
|
|
*/
|
||
|
|
public abstract void onError();
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Called when the recognition is paused temporarily for some reason.
|
||
|
|
* This is an informational callback, and the clients shouldn't be doing anything here
|
||
|
|
* except showing an indication on their UI if they have to.
|
||
|
|
*/
|
||
|
|
public abstract void onRecognitionPaused();
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Called when the recognition is resumed after it was temporarily paused.
|
||
|
|
* This is an informational callback, and the clients shouldn't be doing anything here
|
||
|
|
* except showing an indication on their UI if they have to.
|
||
|
|
*/
|
||
|
|
public abstract void onRecognitionResumed();
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* This class should be constructed by the {@link SoundTriggerManager}.
|
||
|
|
* @hide
|
||
|
|
*/
|
||
|
|
SoundTriggerDetector(ISoundTriggerSession soundTriggerSession,
|
||
|
|
@NonNull GenericSoundModel soundModel,
|
||
|
|
@NonNull Callback callback, @Nullable Handler handler) {
|
||
|
|
mSoundTriggerSession = soundTriggerSession;
|
||
|
|
mSoundModel = soundModel;
|
||
|
|
mCallback = callback;
|
||
|
|
if (handler == null) {
|
||
|
|
mHandler = new MyHandler();
|
||
|
|
} else {
|
||
|
|
mHandler = new MyHandler(handler.getLooper());
|
||
|
|
}
|
||
|
|
mRecognitionCallback = new RecognitionCallback();
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Starts recognition on the associated sound model. Result is indicated via the
|
||
|
|
* {@link Callback}.
|
||
|
|
* @deprecated use {@link SoundTriggerManager} directly
|
||
|
|
* @return Indicates whether the call succeeded or not.
|
||
|
|
*/
|
||
|
|
@Deprecated
|
||
|
|
@RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER)
|
||
|
|
public boolean startRecognition(@RecognitionFlags int recognitionFlags) {
|
||
|
|
if (DBG) {
|
||
|
|
Slog.d(TAG, "startRecognition()");
|
||
|
|
}
|
||
|
|
boolean captureTriggerAudio =
|
||
|
|
(recognitionFlags & RECOGNITION_FLAG_CAPTURE_TRIGGER_AUDIO) != 0;
|
||
|
|
|
||
|
|
boolean allowMultipleTriggers =
|
||
|
|
(recognitionFlags & RECOGNITION_FLAG_ALLOW_MULTIPLE_TRIGGERS) != 0;
|
||
|
|
|
||
|
|
boolean runInBatterySaver = (recognitionFlags & RECOGNITION_FLAG_RUN_IN_BATTERY_SAVER) != 0;
|
||
|
|
|
||
|
|
int audioCapabilities = 0;
|
||
|
|
if ((recognitionFlags & RECOGNITION_FLAG_ENABLE_AUDIO_ECHO_CANCELLATION) != 0) {
|
||
|
|
audioCapabilities |= SoundTrigger.ModuleProperties.AUDIO_CAPABILITY_ECHO_CANCELLATION;
|
||
|
|
}
|
||
|
|
if ((recognitionFlags & RECOGNITION_FLAG_ENABLE_AUDIO_NOISE_SUPPRESSION) != 0) {
|
||
|
|
audioCapabilities |= SoundTrigger.ModuleProperties.AUDIO_CAPABILITY_NOISE_SUPPRESSION;
|
||
|
|
}
|
||
|
|
|
||
|
|
int status;
|
||
|
|
try {
|
||
|
|
status = mSoundTriggerSession.startRecognition(mSoundModel,
|
||
|
|
mRecognitionCallback, new RecognitionConfig(captureTriggerAudio,
|
||
|
|
allowMultipleTriggers, null, null, audioCapabilities),
|
||
|
|
runInBatterySaver);
|
||
|
|
} catch (RemoteException e) {
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
return status == STATUS_OK;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Stops recognition for the associated model.
|
||
|
|
* @deprecated use {@link SoundTriggerManager} directly
|
||
|
|
*/
|
||
|
|
@Deprecated
|
||
|
|
@RequiresPermission(android.Manifest.permission.MANAGE_SOUND_TRIGGER)
|
||
|
|
public boolean stopRecognition() {
|
||
|
|
int status = STATUS_OK;
|
||
|
|
try {
|
||
|
|
status = mSoundTriggerSession.stopRecognition(new ParcelUuid(mSoundModel.getUuid()),
|
||
|
|
mRecognitionCallback);
|
||
|
|
} catch (RemoteException e) {
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
return status == STATUS_OK;
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* @hide
|
||
|
|
*/
|
||
|
|
public void dump(String prefix, PrintWriter pw) {
|
||
|
|
synchronized (mLock) {
|
||
|
|
// TODO: Dump useful debug information.
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Callback that handles events from the lower sound trigger layer.
|
||
|
|
*
|
||
|
|
* Note that these callbacks will be called synchronously from the SoundTriggerService
|
||
|
|
* layer and thus should do minimal work (such as sending a message on a handler to do
|
||
|
|
* the real work).
|
||
|
|
* @hide
|
||
|
|
*/
|
||
|
|
private class RecognitionCallback extends IRecognitionStatusCallback.Stub {
|
||
|
|
|
||
|
|
/**
|
||
|
|
* @hide
|
||
|
|
*/
|
||
|
|
@Override
|
||
|
|
public void onGenericSoundTriggerDetected(SoundTrigger.GenericRecognitionEvent event) {
|
||
|
|
Slog.d(TAG, "onGenericSoundTriggerDetected()" + event);
|
||
|
|
Message.obtain(mHandler,
|
||
|
|
MSG_SOUND_TRIGGER_DETECTED,
|
||
|
|
new EventPayload(event.triggerInData, event.captureAvailable,
|
||
|
|
event.captureFormat, event.captureSession, event.data))
|
||
|
|
.sendToTarget();
|
||
|
|
}
|
||
|
|
|
||
|
|
@Override
|
||
|
|
public void onKeyphraseDetected(SoundTrigger.KeyphraseRecognitionEvent event) {
|
||
|
|
Slog.e(TAG, "Ignoring onKeyphraseDetected() called for " + event);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* @hide
|
||
|
|
*/
|
||
|
|
@Override
|
||
|
|
public void onRecognitionPaused() {
|
||
|
|
Slog.d(TAG, "onRecognitionPaused()");
|
||
|
|
mHandler.sendEmptyMessage(MSG_DETECTION_PAUSE);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* @hide
|
||
|
|
*/
|
||
|
|
@Override
|
||
|
|
public void onRecognitionResumed() {
|
||
|
|
Slog.d(TAG, "onRecognitionResumed()");
|
||
|
|
mHandler.sendEmptyMessage(MSG_DETECTION_RESUME);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* @hide
|
||
|
|
*/
|
||
|
|
@Override
|
||
|
|
public void onPreempted() {
|
||
|
|
Slog.d(TAG, "onPreempted()");
|
||
|
|
mHandler.sendEmptyMessage(MSG_DETECTION_ERROR);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* @hide
|
||
|
|
*/
|
||
|
|
@Override
|
||
|
|
public void onModuleDied() {
|
||
|
|
Slog.d(TAG, "onModuleDied()");
|
||
|
|
mHandler.sendEmptyMessage(MSG_DETECTION_ERROR);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* @hide
|
||
|
|
*/
|
||
|
|
@Override
|
||
|
|
public void onResumeFailed(int status) {
|
||
|
|
Slog.d(TAG, "onResumeFailed()" + status);
|
||
|
|
mHandler.sendEmptyMessage(MSG_DETECTION_ERROR);
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* @hide
|
||
|
|
*/
|
||
|
|
@Override
|
||
|
|
public void onPauseFailed(int status) {
|
||
|
|
Slog.d(TAG, "onPauseFailed()" + status);
|
||
|
|
mHandler.sendEmptyMessage(MSG_DETECTION_ERROR);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private class MyHandler extends Handler {
|
||
|
|
|
||
|
|
MyHandler() {
|
||
|
|
super();
|
||
|
|
}
|
||
|
|
|
||
|
|
MyHandler(Looper looper) {
|
||
|
|
super(looper);
|
||
|
|
}
|
||
|
|
|
||
|
|
@Override
|
||
|
|
public void handleMessage(Message msg) {
|
||
|
|
if (mCallback == null) {
|
||
|
|
Slog.w(TAG, "Received message: " + msg.what + " for NULL callback.");
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
switch (msg.what) {
|
||
|
|
case MSG_SOUND_TRIGGER_DETECTED:
|
||
|
|
mCallback.onDetected((EventPayload) msg.obj);
|
||
|
|
break;
|
||
|
|
case MSG_DETECTION_ERROR:
|
||
|
|
mCallback.onError();
|
||
|
|
break;
|
||
|
|
case MSG_DETECTION_PAUSE:
|
||
|
|
mCallback.onRecognitionPaused();
|
||
|
|
break;
|
||
|
|
case MSG_DETECTION_RESUME:
|
||
|
|
mCallback.onRecognitionResumed();
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
super.handleMessage(msg);
|
||
|
|
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|