461 lines
18 KiB
Java
461 lines
18 KiB
Java
![]() |
/*
|
|||
|
* Copyright (C) 2020 The Android Open Source Project
|
|||
|
*
|
|||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|||
|
* you may not use this file except in compliance with the License.
|
|||
|
* You may obtain a copy of the License at
|
|||
|
*
|
|||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
|
*
|
|||
|
* Unless required by applicable law or agreed to in writing, software
|
|||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|||
|
* See the License for the specific language governing permissions and
|
|||
|
* limitations under the License.
|
|||
|
*/
|
|||
|
|
|||
|
package android.service.voice;
|
|||
|
|
|||
|
import static java.util.Objects.requireNonNull;
|
|||
|
|
|||
|
import android.annotation.DurationMillisLong;
|
|||
|
import android.annotation.FlaggedApi;
|
|||
|
import android.annotation.IntDef;
|
|||
|
import android.annotation.NonNull;
|
|||
|
import android.annotation.Nullable;
|
|||
|
import android.annotation.SdkConstant;
|
|||
|
import android.annotation.SuppressLint;
|
|||
|
import android.annotation.SystemApi;
|
|||
|
import android.annotation.TestApi;
|
|||
|
import android.app.Service;
|
|||
|
import android.content.ContentCaptureOptions;
|
|||
|
import android.content.Context;
|
|||
|
import android.content.Intent;
|
|||
|
import android.hardware.soundtrigger.SoundTrigger;
|
|||
|
import android.media.AudioFormat;
|
|||
|
import android.media.AudioSystem;
|
|||
|
import android.os.IBinder;
|
|||
|
import android.os.IRemoteCallback;
|
|||
|
import android.os.ParcelFileDescriptor;
|
|||
|
import android.os.PersistableBundle;
|
|||
|
import android.os.RemoteException;
|
|||
|
import android.os.SharedMemory;
|
|||
|
import android.speech.IRecognitionServiceManager;
|
|||
|
import android.util.Log;
|
|||
|
import android.view.contentcapture.ContentCaptureManager;
|
|||
|
import android.view.contentcapture.IContentCaptureManager;
|
|||
|
|
|||
|
import java.lang.annotation.Documented;
|
|||
|
import java.lang.annotation.Retention;
|
|||
|
import java.lang.annotation.RetentionPolicy;
|
|||
|
import java.util.Locale;
|
|||
|
import java.util.function.IntConsumer;
|
|||
|
|
|||
|
/**
|
|||
|
* Implemented by an application that wants to offer detection for hotword. The service can be used
|
|||
|
* for both DSP and non-DSP detectors.
|
|||
|
*
|
|||
|
* The system will bind an application's {@link VoiceInteractionService} first. When {@link
|
|||
|
* VoiceInteractionService#createHotwordDetector(PersistableBundle, SharedMemory,
|
|||
|
* HotwordDetector.Callback)} or {@link VoiceInteractionService#createAlwaysOnHotwordDetector(
|
|||
|
* String, Locale, PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)} is called,
|
|||
|
* the system will bind application's {@link HotwordDetectionService}. Either on a hardware
|
|||
|
* trigger or on request from the {@link VoiceInteractionService}, the system calls into the
|
|||
|
* {@link HotwordDetectionService} to request detection. The {@link HotwordDetectionService} then
|
|||
|
* uses {@link Callback#onDetected(HotwordDetectedResult)} to inform the system that a relevant
|
|||
|
* keyphrase was detected, or if applicable uses {@link Callback#onRejected(HotwordRejectedResult)}
|
|||
|
* to inform the system that a keyphrase was not detected. The system then relays this result to
|
|||
|
* the {@link VoiceInteractionService} through {@link HotwordDetector.Callback}.
|
|||
|
*
|
|||
|
* Note: Methods in this class may be called concurrently
|
|||
|
*
|
|||
|
* @hide
|
|||
|
*/
|
|||
|
@SystemApi
|
|||
|
public abstract class HotwordDetectionService extends Service
|
|||
|
implements SandboxedDetectionInitializer {
|
|||
|
private static final String TAG = "HotwordDetectionService";
|
|||
|
private static final boolean DBG = false;
|
|||
|
|
|||
|
private static final long UPDATE_TIMEOUT_MILLIS = 20000;
|
|||
|
|
|||
|
/**
|
|||
|
* The PersistableBundle options key used in {@link #onDetect(ParcelFileDescriptor, AudioFormat,
|
|||
|
* PersistableBundle, Callback)} to indicate whether the system will close the audio stream
|
|||
|
* after {@code Callback} is invoked.
|
|||
|
*/
|
|||
|
@FlaggedApi(android.app.wearable.Flags.FLAG_ENABLE_HOTWORD_WEARABLE_SENSING_API)
|
|||
|
public static final String KEY_SYSTEM_WILL_CLOSE_AUDIO_STREAM_AFTER_CALLBACK =
|
|||
|
"android.service.voice.HotwordDetectionService."
|
|||
|
+ "KEY_SYSTEM_WILL_CLOSE_AUDIO_STREAM_AFTER_CALLBACK";
|
|||
|
|
|||
|
/**
|
|||
|
* Feature flag for Attention Service.
|
|||
|
*
|
|||
|
* @hide
|
|||
|
*/
|
|||
|
@TestApi
|
|||
|
public static final boolean ENABLE_PROXIMITY_RESULT = true;
|
|||
|
|
|||
|
/**
|
|||
|
* Indicates that the updated status is successful.
|
|||
|
*
|
|||
|
* @deprecated Replaced with
|
|||
|
* {@link SandboxedDetectionInitializer#INITIALIZATION_STATUS_SUCCESS}
|
|||
|
*/
|
|||
|
@Deprecated
|
|||
|
public static final int INITIALIZATION_STATUS_SUCCESS =
|
|||
|
SandboxedDetectionInitializer.INITIALIZATION_STATUS_SUCCESS;
|
|||
|
|
|||
|
/**
|
|||
|
* Indicates that the callback wasn’t invoked within the timeout.
|
|||
|
* This is used by system.
|
|||
|
*
|
|||
|
* @deprecated Replaced with
|
|||
|
* {@link SandboxedDetectionInitializer#INITIALIZATION_STATUS_UNKNOWN}
|
|||
|
*/
|
|||
|
@Deprecated
|
|||
|
public static final int INITIALIZATION_STATUS_UNKNOWN =
|
|||
|
SandboxedDetectionInitializer.INITIALIZATION_STATUS_UNKNOWN;
|
|||
|
|
|||
|
/**
|
|||
|
* Source for the given audio stream.
|
|||
|
*
|
|||
|
* @hide
|
|||
|
*/
|
|||
|
@Documented
|
|||
|
@Retention(RetentionPolicy.SOURCE)
|
|||
|
@IntDef({
|
|||
|
AUDIO_SOURCE_MICROPHONE,
|
|||
|
AUDIO_SOURCE_EXTERNAL
|
|||
|
})
|
|||
|
@interface AudioSource {}
|
|||
|
|
|||
|
/** @hide */
|
|||
|
public static final int AUDIO_SOURCE_MICROPHONE = 1;
|
|||
|
/** @hide */
|
|||
|
public static final int AUDIO_SOURCE_EXTERNAL = 2;
|
|||
|
|
|||
|
/**
|
|||
|
* The {@link Intent} that must be declared as handled by the service.
|
|||
|
* To be supported, the service must also require the
|
|||
|
* {@link android.Manifest.permission#BIND_HOTWORD_DETECTION_SERVICE} permission so
|
|||
|
* that other applications can not abuse it.
|
|||
|
*/
|
|||
|
@SdkConstant(SdkConstant.SdkConstantType.SERVICE_ACTION)
|
|||
|
public static final String SERVICE_INTERFACE =
|
|||
|
"android.service.voice.HotwordDetectionService";
|
|||
|
|
|||
|
@Nullable
|
|||
|
private ContentCaptureManager mContentCaptureManager;
|
|||
|
@Nullable
|
|||
|
private IRecognitionServiceManager mIRecognitionServiceManager;
|
|||
|
|
|||
|
private final ISandboxedDetectionService mInterface = new ISandboxedDetectionService.Stub() {
|
|||
|
@Override
|
|||
|
public void detectFromDspSource(
|
|||
|
SoundTrigger.KeyphraseRecognitionEvent event,
|
|||
|
AudioFormat audioFormat,
|
|||
|
long timeoutMillis,
|
|||
|
IDspHotwordDetectionCallback callback)
|
|||
|
throws RemoteException {
|
|||
|
if (DBG) {
|
|||
|
Log.d(TAG, "#detectFromDspSource");
|
|||
|
}
|
|||
|
HotwordDetectionService.this.onDetect(
|
|||
|
new AlwaysOnHotwordDetector.EventPayload.Builder(event).build(),
|
|||
|
timeoutMillis,
|
|||
|
new Callback(callback));
|
|||
|
}
|
|||
|
|
|||
|
@Override
|
|||
|
public void updateState(PersistableBundle options, SharedMemory sharedMemory,
|
|||
|
IRemoteCallback callback) throws RemoteException {
|
|||
|
Log.v(TAG, "#updateState" + (callback != null ? " with callback" : ""));
|
|||
|
HotwordDetectionService.this.onUpdateStateInternal(
|
|||
|
options,
|
|||
|
sharedMemory,
|
|||
|
callback);
|
|||
|
}
|
|||
|
|
|||
|
@Override
|
|||
|
public void detectFromMicrophoneSource(
|
|||
|
ParcelFileDescriptor audioStream,
|
|||
|
@AudioSource int audioSource,
|
|||
|
AudioFormat audioFormat,
|
|||
|
PersistableBundle options,
|
|||
|
IDspHotwordDetectionCallback callback)
|
|||
|
throws RemoteException {
|
|||
|
if (DBG) {
|
|||
|
Log.d(TAG, "#detectFromMicrophoneSource");
|
|||
|
}
|
|||
|
switch (audioSource) {
|
|||
|
case AUDIO_SOURCE_MICROPHONE:
|
|||
|
HotwordDetectionService.this.onDetect(
|
|||
|
new Callback(callback));
|
|||
|
break;
|
|||
|
case AUDIO_SOURCE_EXTERNAL:
|
|||
|
HotwordDetectionService.this.onDetect(
|
|||
|
audioStream,
|
|||
|
audioFormat,
|
|||
|
options,
|
|||
|
new Callback(callback));
|
|||
|
break;
|
|||
|
default:
|
|||
|
Log.i(TAG, "Unsupported audio source " + audioSource);
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
@Override
|
|||
|
public void detectWithVisualSignals(
|
|||
|
IDetectorSessionVisualQueryDetectionCallback callback) {
|
|||
|
throw new UnsupportedOperationException("Not supported by HotwordDetectionService");
|
|||
|
}
|
|||
|
|
|||
|
@Override
|
|||
|
public void updateAudioFlinger(IBinder audioFlinger) {
|
|||
|
AudioSystem.setAudioFlingerBinder(audioFlinger);
|
|||
|
}
|
|||
|
|
|||
|
@Override
|
|||
|
public void updateContentCaptureManager(IContentCaptureManager manager,
|
|||
|
ContentCaptureOptions options) {
|
|||
|
mContentCaptureManager = new ContentCaptureManager(
|
|||
|
HotwordDetectionService.this, manager, options);
|
|||
|
}
|
|||
|
|
|||
|
@Override
|
|||
|
public void updateRecognitionServiceManager(IRecognitionServiceManager manager) {
|
|||
|
mIRecognitionServiceManager = manager;
|
|||
|
}
|
|||
|
|
|||
|
@Override
|
|||
|
public void ping(IRemoteCallback callback) throws RemoteException {
|
|||
|
callback.sendResult(null);
|
|||
|
}
|
|||
|
|
|||
|
@Override
|
|||
|
public void stopDetection() {
|
|||
|
HotwordDetectionService.this.onStopDetection();
|
|||
|
}
|
|||
|
|
|||
|
@Override
|
|||
|
public void registerRemoteStorageService(IDetectorSessionStorageService
|
|||
|
detectorSessionStorageService) {
|
|||
|
throw new UnsupportedOperationException("Hotword cannot access files from the disk.");
|
|||
|
}
|
|||
|
};
|
|||
|
|
|||
|
@Override
|
|||
|
@Nullable
|
|||
|
public final IBinder onBind(@NonNull Intent intent) {
|
|||
|
if (SERVICE_INTERFACE.equals(intent.getAction())) {
|
|||
|
return mInterface.asBinder();
|
|||
|
}
|
|||
|
Log.w(TAG, "Tried to bind to wrong intent (should be " + SERVICE_INTERFACE + ": "
|
|||
|
+ intent);
|
|||
|
return null;
|
|||
|
}
|
|||
|
|
|||
|
@Override
|
|||
|
@SuppressLint("OnNameExpected")
|
|||
|
public @Nullable Object getSystemService(@ServiceName @NonNull String name) {
|
|||
|
if (Context.CONTENT_CAPTURE_MANAGER_SERVICE.equals(name)) {
|
|||
|
return mContentCaptureManager;
|
|||
|
} else if (Context.SPEECH_RECOGNITION_SERVICE.equals(name)
|
|||
|
&& mIRecognitionServiceManager != null) {
|
|||
|
return mIRecognitionServiceManager.asBinder();
|
|||
|
} else {
|
|||
|
return super.getSystemService(name);
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
/**
|
|||
|
* Returns the maximum number of initialization status for some application specific failed
|
|||
|
* reasons.
|
|||
|
*
|
|||
|
* Note: The value 0 is reserved for success.
|
|||
|
*
|
|||
|
* @hide
|
|||
|
* @deprecated Replaced with
|
|||
|
* {@link SandboxedDetectionInitializer#getMaxCustomInitializationStatus()}
|
|||
|
*/
|
|||
|
@SystemApi
|
|||
|
@Deprecated
|
|||
|
public static int getMaxCustomInitializationStatus() {
|
|||
|
return MAXIMUM_NUMBER_OF_INITIALIZATION_STATUS_CUSTOM_ERROR;
|
|||
|
}
|
|||
|
|
|||
|
/**
|
|||
|
* Called when the device hardware (such as a DSP) detected the hotword, to request second stage
|
|||
|
* validation before handing over the audio to the {@link AlwaysOnHotwordDetector}.
|
|||
|
*
|
|||
|
* <p>After {@code callback} is invoked or {@code timeoutMillis} has passed, and invokes the
|
|||
|
* appropriate {@link AlwaysOnHotwordDetector.Callback callback}.
|
|||
|
*
|
|||
|
* <p>When responding to a detection event, the
|
|||
|
* {@link HotwordDetectedResult#getHotwordPhraseId()} must match a keyphrase ID listed
|
|||
|
* in the eventPayload's
|
|||
|
* {@link AlwaysOnHotwordDetector.EventPayload#getKeyphraseRecognitionExtras()} list. This is
|
|||
|
* forcing the intention of the {@link HotwordDetectionService} to validate an event from the
|
|||
|
* voice engine and not augment its result.
|
|||
|
*
|
|||
|
* @param eventPayload Payload data for the hardware detection event. This may contain the
|
|||
|
* trigger audio, if requested when calling
|
|||
|
* {@link AlwaysOnHotwordDetector#startRecognition(int)}.
|
|||
|
* Each {@link AlwaysOnHotwordDetector} will be associated with at minimum a unique
|
|||
|
* keyphrase ID indicated by
|
|||
|
* {@link AlwaysOnHotwordDetector.EventPayload#getKeyphraseRecognitionExtras()}[0].
|
|||
|
* Any extra
|
|||
|
* {@link android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra}'s
|
|||
|
* in the eventPayload represent additional phrases detected by the voice engine.
|
|||
|
* @param timeoutMillis Timeout in milliseconds for the operation to invoke the callback. If
|
|||
|
* the application fails to abide by the timeout, system will close the
|
|||
|
* microphone and cancel the operation.
|
|||
|
* @param callback The callback to use for responding to the detection request.
|
|||
|
*
|
|||
|
* @hide
|
|||
|
*/
|
|||
|
@SystemApi
|
|||
|
public void onDetect(
|
|||
|
@NonNull AlwaysOnHotwordDetector.EventPayload eventPayload,
|
|||
|
@DurationMillisLong long timeoutMillis,
|
|||
|
@NonNull Callback callback) {
|
|||
|
// TODO: Add a helpful error message.
|
|||
|
throw new UnsupportedOperationException();
|
|||
|
}
|
|||
|
|
|||
|
/**
|
|||
|
* Called when the {@link VoiceInteractionService#createAlwaysOnHotwordDetector(String, Locale,
|
|||
|
* PersistableBundle, SharedMemory, AlwaysOnHotwordDetector.Callback)} or
|
|||
|
* {@link AlwaysOnHotwordDetector#updateState(PersistableBundle, SharedMemory)} requests an
|
|||
|
* update of the hotword detection parameters.
|
|||
|
*
|
|||
|
* {@inheritDoc}
|
|||
|
* @hide
|
|||
|
*/
|
|||
|
@Override
|
|||
|
@SystemApi
|
|||
|
public void onUpdateState(
|
|||
|
@Nullable PersistableBundle options,
|
|||
|
@Nullable SharedMemory sharedMemory,
|
|||
|
@DurationMillisLong long callbackTimeoutMillis,
|
|||
|
@Nullable IntConsumer statusCallback) {}
|
|||
|
|
|||
|
/**
|
|||
|
* Called when the {@link VoiceInteractionService} requests that this service
|
|||
|
* {@link HotwordDetector#startRecognition() start} hotword recognition on audio coming directly
|
|||
|
* from the device microphone.
|
|||
|
* <p>
|
|||
|
* On successful detection of a hotword, call
|
|||
|
* {@link Callback#onDetected(HotwordDetectedResult)}.
|
|||
|
*
|
|||
|
* @param callback The callback to use for responding to the detection request.
|
|||
|
* {@link Callback#onRejected(HotwordRejectedResult) callback.onRejected} cannot be used here.
|
|||
|
*/
|
|||
|
public void onDetect(@NonNull Callback callback) {
|
|||
|
// TODO: Add a helpful error message.
|
|||
|
throw new UnsupportedOperationException();
|
|||
|
}
|
|||
|
|
|||
|
/**
|
|||
|
* Called when the {@link VoiceInteractionService} requests that this service
|
|||
|
* {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat,
|
|||
|
* PersistableBundle)} run} hotword recognition on audio coming from an external connected
|
|||
|
* microphone.
|
|||
|
*
|
|||
|
* <p>Upon invoking the {@code callback}, the system will send the detection result to
|
|||
|
* the {@link HotwordDetector}'s callback. If {@code
|
|||
|
* options.getBoolean(KEY_SYSTEM_WILL_CLOSE_AUDIO_STREAM_AFTER_CALLBACK, true)} returns true,
|
|||
|
* the system will also close the {@code audioStream} after {@code callback} is invoked.
|
|||
|
*
|
|||
|
* @param audioStream Stream containing audio bytes returned from a microphone
|
|||
|
* @param audioFormat Format of the supplied audio
|
|||
|
* @param options Options supporting detection, such as configuration specific to the source of
|
|||
|
* the audio, provided through
|
|||
|
* {@link HotwordDetector#startRecognition(ParcelFileDescriptor, AudioFormat,
|
|||
|
* PersistableBundle)}.
|
|||
|
* @param callback The callback to use for responding to the detection request.
|
|||
|
*/
|
|||
|
public void onDetect(
|
|||
|
@NonNull ParcelFileDescriptor audioStream,
|
|||
|
@NonNull AudioFormat audioFormat,
|
|||
|
@Nullable PersistableBundle options,
|
|||
|
@NonNull Callback callback) {
|
|||
|
// TODO: Add a helpful error message.
|
|||
|
throw new UnsupportedOperationException();
|
|||
|
}
|
|||
|
|
|||
|
private void onUpdateStateInternal(@Nullable PersistableBundle options,
|
|||
|
@Nullable SharedMemory sharedMemory, IRemoteCallback callback) {
|
|||
|
IntConsumer intConsumer =
|
|||
|
SandboxedDetectionInitializer.createInitializationStatusConsumer(callback);
|
|||
|
onUpdateState(options, sharedMemory, UPDATE_TIMEOUT_MILLIS, intConsumer);
|
|||
|
}
|
|||
|
|
|||
|
/**
|
|||
|
* Called when the {@link VoiceInteractionService}
|
|||
|
* {@link HotwordDetector#stopRecognition() requests} that hotword recognition be stopped.
|
|||
|
* <p>
|
|||
|
* Any open {@link android.media.AudioRecord} should be closed here.
|
|||
|
*/
|
|||
|
public void onStopDetection() {
|
|||
|
}
|
|||
|
|
|||
|
/**
|
|||
|
* Callback for returning the detection result.
|
|||
|
*
|
|||
|
* @hide
|
|||
|
*/
|
|||
|
@SystemApi
|
|||
|
public static final class Callback {
|
|||
|
// TODO: consider making the constructor a test api for testing purpose
|
|||
|
private final IDspHotwordDetectionCallback mRemoteCallback;
|
|||
|
|
|||
|
private Callback(IDspHotwordDetectionCallback remoteCallback) {
|
|||
|
mRemoteCallback = remoteCallback;
|
|||
|
}
|
|||
|
|
|||
|
/**
|
|||
|
* Informs the {@link HotwordDetector} that the keyphrase was detected.
|
|||
|
*
|
|||
|
* @param result Info about the detection result. This is provided to the
|
|||
|
* {@link HotwordDetector}.
|
|||
|
*/
|
|||
|
public void onDetected(@NonNull HotwordDetectedResult result) {
|
|||
|
requireNonNull(result);
|
|||
|
final PersistableBundle persistableBundle = result.getExtras();
|
|||
|
if (!persistableBundle.isEmpty() && HotwordDetectedResult.getParcelableSize(
|
|||
|
persistableBundle) > HotwordDetectedResult.getMaxBundleSize()) {
|
|||
|
throw new IllegalArgumentException(
|
|||
|
"The bundle size of result is larger than max bundle size ("
|
|||
|
+ HotwordDetectedResult.getMaxBundleSize()
|
|||
|
+ ") of HotwordDetectedResult");
|
|||
|
}
|
|||
|
try {
|
|||
|
mRemoteCallback.onDetected(result);
|
|||
|
} catch (RemoteException e) {
|
|||
|
throw e.rethrowFromSystemServer();
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
/**
|
|||
|
* Informs the {@link HotwordDetector} that the keyphrase was not detected.
|
|||
|
* <p>
|
|||
|
* This cannot not be used when recognition is done through
|
|||
|
* {@link #onDetect(ParcelFileDescriptor, AudioFormat, Callback)}.
|
|||
|
*
|
|||
|
* @param result Info about the second stage detection result. This is provided to
|
|||
|
* the {@link HotwordDetector}.
|
|||
|
*/
|
|||
|
public void onRejected(@NonNull HotwordRejectedResult result) {
|
|||
|
requireNonNull(result);
|
|||
|
try {
|
|||
|
mRemoteCallback.onRejected(result);
|
|||
|
} catch (RemoteException e) {
|
|||
|
throw e.rethrowFromSystemServer();
|
|||
|
}
|
|||
|
}
|
|||
|
}
|
|||
|
}
|