Add android.media.audio.eraser am: 477ce0948e Original change: https://android-review.googlesource.com/c/platform/system/hardware/interfaces/+/3388267 Change-Id: I998bf8a1f980a3e1f688ea0df6f0de97b5372342 Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
diff --git a/media/Android.bp b/media/Android.bp index a70d5f8..4551f2e 100644 --- a/media/Android.bp +++ b/media/Android.bp
@@ -93,7 +93,6 @@ "aidl/android/media/audio/common/MicrophoneDynamicInfo.aidl", "aidl/android/media/audio/common/MicrophoneInfo.aidl", "aidl/android/media/audio/common/PcmType.aidl", - "aidl/android/media/audio/common/SoundClassification.aidl", "aidl/android/media/audio/common/Spatialization.aidl", "aidl/android/media/audio/common/Void.aidl", ], @@ -340,3 +339,110 @@ latest_android_media_soundtrigger_types, ], } + +aidl_interface { + name: "android.media.audio.eraser.types", + vendor_available: true, + host_supported: true, + flags: [ + "-Werror", + "-Weverything", + ], + local_include_dir: "aidl", + srcs: [ + "aidl/android/media/audio/eraser/Capability.aidl", + "aidl/android/media/audio/eraser/Classification.aidl", + "aidl/android/media/audio/eraser/ClassificationConfig.aidl", + "aidl/android/media/audio/eraser/ClassificationMetadata.aidl", + "aidl/android/media/audio/eraser/ClassificationMetadataList.aidl", + "aidl/android/media/audio/eraser/ClassifierCapability.aidl", + "aidl/android/media/audio/eraser/Configuration.aidl", + "aidl/android/media/audio/eraser/IEraserCallback.aidl", + "aidl/android/media/audio/eraser/Mode.aidl", + "aidl/android/media/audio/eraser/RemixerCapability.aidl", + "aidl/android/media/audio/eraser/SeparatorCapability.aidl", + "aidl/android/media/audio/eraser/SoundClassification.aidl", + ], + stability: "vintf", + backend: { + cpp: { + enabled: true, + }, + java: { + sdk_version: "module_current", + }, + }, + imports: [ + latest_android_media_audio_common_types, + ], + frozen: false, +} + +// Note: This should always be one version ahead of the last frozen version +latest_android_media_audio_eraser_types = "android.media.audio.eraser.types-V1" + +cc_defaults { + name: "latest_android_media_audio_eraser_types_cpp_shared", + shared_libs: [ + latest_android_media_audio_eraser_types + "-cpp", + ], +} + +cc_defaults { + name: "latest_android_media_audio_eraser_types_cpp_export_shared", + defaults: [ + "latest_android_media_audio_eraser_types_cpp_shared", + ], + export_shared_lib_headers: [ + latest_android_media_audio_eraser_types + "-cpp", + ], +} + +cc_defaults { + name: "latest_android_media_audio_eraser_types_cpp_static", + static_libs: [ + latest_android_media_audio_eraser_types + "-cpp", + ], +} + +cc_defaults { + name: "latest_android_media_audio_eraser_types_cpp_export_static", + defaults: [ + "latest_android_media_audio_eraser_types_cpp_static", + ], + export_static_lib_headers: [ + latest_android_media_audio_eraser_types + "-cpp", + ], +} + +cc_defaults { + name: "latest_android_media_audio_eraser_types_ndk_shared", + shared_libs: [ + latest_android_media_audio_eraser_types + "-ndk", + ], +} + +cc_defaults { + name: "latest_android_media_audio_eraser_types_ndk_static", + static_libs: [ + latest_android_media_audio_eraser_types + "-ndk", + ], +} + +cc_defaults { + name: "latest_android_media_audio_eraser_types_cpp_target_shared", + target: { + android: { + shared_libs: [ + latest_android_media_audio_eraser_types + "-cpp", + ], + }, + }, +} + +aidl_interface_defaults { + name: "latest_android_media_audio_eraser_types_import_interface", + imports: [ + latest_android_media_audio_eraser_types, + ], +}
diff --git a/media/aidl/android/media/audio/eraser/Capability.aidl b/media/aidl/android/media/audio/eraser/Capability.aidl new file mode 100644 index 0000000..a7627e5 --- /dev/null +++ b/media/aidl/android/media/audio/eraser/Capability.aidl
@@ -0,0 +1,80 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.audio.eraser; + +import android.media.audio.common.AudioChannelLayout; +import android.media.audio.eraser.ClassifierCapability; +import android.media.audio.eraser.Mode; +import android.media.audio.eraser.RemixerCapability; +import android.media.audio.eraser.SeparatorCapability; + +/** + * Represents the capability of an audio eraser. + * + * This parcelable defines the supported input/output data formats, available work modes, and the + * specific capabilities of the sound classifier, separator, and remixer components within the + * eraser effect. + */ +@JavaDerive(equals=true, toString=true) +@VintfStability +parcelable Capability { + /** + * List of supported sample rates for the eraser. + * + * The output audio sample rate will be the same as the input. + */ + int[] sampleRates; + + /** + * List of supported channel layouts for the eraser. + * + * The output audio channel layout will be the same as the input. + */ + AudioChannelLayout[] channelLayouts; + + /** + * List of supported work modes. + * + * Defines the different operational modes (e.g., `ERASER`, `CLASSIFIER`) that the eraser can + * work in. + */ + Mode[] modes; + + /** + * Separator capability. + * + * Specifies the capabilities of the sound separator component within the eraser effect, + * including the maximum number of sound sources it can separate. + */ + SeparatorCapability separator; + + /** + * Classifier capability. + * + * Specifies the capabilities of the sound classifier component within the eraser effect, + * including the sound classifications it can detect. + */ + ClassifierCapability classifier; + + /** + * Remixer capability. + * + * Specifies the capabilities of the sound remixer component within the eraser effect, + * including the gainFactor range supported. + */ + RemixerCapability remixer; +}
diff --git a/media/aidl/android/media/audio/eraser/Classification.aidl b/media/aidl/android/media/audio/eraser/Classification.aidl new file mode 100644 index 0000000..16aa1ce --- /dev/null +++ b/media/aidl/android/media/audio/eraser/Classification.aidl
@@ -0,0 +1,37 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.audio.eraser; + +import android.media.audio.eraser.SoundClassification; + +/** + * Represents a sound classification category. + * + * The classification includes the top-level sound category based on the AudioSet ontology. + */ +@JavaDerive(equals=true, toString=true) +@VintfStability +parcelable Classification { + /** + * The top-level sounds classification supported. + * + * This field specifies the primary sound category that this classification represents, + * as defined in the AudioSet ontology. It helps identify the general type of sound, + * such as HUMAN, ANIMAL, MUSIC, etc. + */ + SoundClassification classification = SoundClassification.HUMAN; +}
diff --git a/media/aidl/android/media/audio/eraser/ClassificationConfig.aidl b/media/aidl/android/media/audio/eraser/ClassificationConfig.aidl new file mode 100644 index 0000000..ab5191e --- /dev/null +++ b/media/aidl/android/media/audio/eraser/ClassificationConfig.aidl
@@ -0,0 +1,125 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.audio.eraser; + +import android.media.audio.eraser.Classification; + +/** + * Configuration for the eraser to apply specific gain adjustments to certain sound classifications. + * + * Gain is applied to the audio signal by scaling the amplitude of the output audio based on the + * classification of the input sound. + * If a classification exists in the configuration list, the remixer applies the specified gain to + * the output audio when the confidence score is higher than `confidenceThreshold`. If a + * classification is not present in the configuration, it is considered to have a gain of 1.0 + * (no gain adjustment). + * If a ClassificationConfig contains an empty classification list, the same threshold and gain + * specified in the ClassificationConfig will be applied to all classifications not explicitly + * configured. + * + * Examples: + * + * 1. {classifications = [{classification = SoundClassification.NATURE}, + * {classification = SoundClassification.ENVIRONMENT}], + * confidenceThreshold = 0.8, + * gainFactor = 0.0} + * + * - If the input audio is classified as NATURE or ENVIRONMENT, with a confidence score higher + * than 0.8, the output audio will be muted. + * - If the classification confidence score is 0.8 or lower, or if the audio is classified + * differently, the output audio remains unchanged. + * + * 2. {classifications = [{classification = SoundClassification.MUSIC}], + * confidenceThreshold = 0.6, + * gainFactor = 0.5} + * + * - If the input audio is classified as MUSIC with a confidence score higher than 0.6, the + * output audio should have a gain factor of 0.5 (reduced by half). + * - If the classification confidence score is 0.6 or lower, or if the audio is classified + * differently, the output audio remains unchanged. + * + * 3. When combined as a list, the eraser can be configured to apply different gainFactor to + * a classifications when confideence score is higher than the corresponding threshold. + * [{classifications = [{classification = SoundClassification.NATURE}], + * confidenceThreshold = 0.8, + * gainFactor = 0.0}, + * {classifications = [{classification = SoundClassification.MUSIC}], + * confidenceThreshold = 0.8, + * gainFactor = 0.6}, + * {classifications = [{classification = SoundClassification.MUSIC}], + * confidenceThreshold = 0.5, + * gainFactor = 0.5}] + * + * - If the input audio is classified as NATURE, and the confidence score is higher than 0.8, + * the output audio classification will be muted (gainFactor = 0.0). + * + * - If the input audio is classified as MUSIC with a confidence score higher than 0.8, the + * output audio classification will have a gain factor of 0.6. If the input audio is + * classified as MUSIC with a confidence score higher than 0.5, the output audio + * classification will have a gain factor of 0.5. + * + * - For all other sound classifications, the audio signal remains unchanged (gainFactor = 1.0). + * + * 4. [{classifications = [{classification = SoundClassification.HUMAN}], + * confidenceThreshold = 0.8, + * gainFactor = 1.0}, + * {classifications = [], + * confidenceThreshold = 0.0, + * gainFactor = 0.5}] + * + * - If the input audio is classified as HUMAN, and the confidence score is higher than 0.8, the + * output audio classification will remains unchanged. + * + * - For all other sound classifications, the audio signal will have a gain factor of 0.5. + * + */ +@JavaDerive(equals=true, toString=true) +@VintfStability +parcelable ClassificationConfig { + /** + * List of sound classifications to which this configuration applies. + * + * Each entry specifies a sound classification (e.g., MUSIC, NATURE) targeted by this + * configuration. + */ + Classification[] classifications; + + /** + * Confidence threshold in the range of [0.0, 1.0], only apply the gainFactor when the + * classifier's confidence score for the specified classifications exceeds this threshold. + * + * Default Value is 0.0 which means apply gain regardless of confidence score. + */ + float confidenceThreshold = 0f; + + /** + * Gain factor to apply to the output audio when the specified classifications are detected. + * Gain factor is applied by multiplying the amplitude of the audio signal by the `gainFactor`. + * + * - A `gainFactor` of `1.0` means no gain adjustment (the original volume is preserved). + * - A `gainFactor` of `0.5` reduces the amplitude of the audio by half. + * - A `gainFactor` of `0.0` mutes the audio. + * - A `gainFactor` > `1.0` amplifies the audio signal, increasing its volume (useful for + * compressor and amplification cases). + * - A `gainFactor` < `0.0` inverts the phase of the audio signal (useful for phase + * cancellation or specific spatial audio manipulation). + * + * The `gainFactor` must be within the `gainFactorRange` defined in `RemixerCapability`, the + * default value is `1.0`. + */ + float gainFactor = 1f; +}
diff --git a/media/aidl/android/media/audio/eraser/ClassificationMetadata.aidl b/media/aidl/android/media/audio/eraser/ClassificationMetadata.aidl new file mode 100644 index 0000000..cc79c37 --- /dev/null +++ b/media/aidl/android/media/audio/eraser/ClassificationMetadata.aidl
@@ -0,0 +1,42 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.audio.eraser; + +import android.media.audio.eraser.Classification; + +/** + * Metadata generated by a sound classification task. + * + * This parcelable contains the classification result for a segment of the audio stream, along with + * a confidence score indicating the certainty of the classification. + */ +@JavaDerive(equals=true, toString=true) +@VintfStability +parcelable ClassificationMetadata { + /** + * Confidence score for the classification, ranging from 0.0 to 1.0. + * + * This score reflects the classifier's confidence in the result, with higher values + * representing greater confidence in the prediction. + */ + float confidenceScore; + + /** + * The classification result, indicating the top-level sound classification. + */ + Classification classification; +}
diff --git a/media/aidl/android/media/audio/eraser/ClassificationMetadataList.aidl b/media/aidl/android/media/audio/eraser/ClassificationMetadataList.aidl new file mode 100644 index 0000000..3a5dd16 --- /dev/null +++ b/media/aidl/android/media/audio/eraser/ClassificationMetadataList.aidl
@@ -0,0 +1,82 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.audio.eraser; + +import android.media.audio.eraser.ClassificationMetadata; + +/** + * List of active `ClassificationMetadata` aligned to a specific timestamp. + * + * A `ClassificationMetadata` is considered active when the `confidenceScore` exceeds the + * `ClassificationConfig.confidenceThreshold`. + * + * The classifier component in the eraser must maintain the active metadata list when an + * `IEraserCallback` is configured and send the list via `onClassifierUpdate` whenever a change + * occurs. + */ + +@JavaDerive(equals=true, toString=true) +@VintfStability +parcelable ClassificationMetadataList { + /** + * Timestamp in milliseconds within the audio stream that this classification result is aligned + * with, the timestamp is calculated with audio frames the eraser effect received, starting + * from the first frame processed by the eraser effect. + * + * The `timeMs` indicates the starting point in the audio stream that the classification results + * in this metadata list apply to. + * Each classifier process window produces a list of `ClassificationMetadata`. The `timeMs` in + * the metadata list always aligns with the start of the window (the starting point of the audio + * segment processed by the classifier). + * In rare cases where the classifier produces an identical list of classifications for + * consecutive windows (including confidence scores), the `onClassifierUpdate` callback will + * only be triggered once for the first process window, with a `timeMs` indicating the start of + * that window. No further `onClassifierUpdate` callbacks will be made for the subsequent + * windows, as there is no meaningful change in the classification results. This avoids + * redundant updates when the classification remains the same across windows. + * + * Client Usage: + * The `timeMs` allows clients to map the classification results back to a specific portion of + * the audio stream. Clients can use this information to synchronize classification results + * with the audio data or other events. Each metadata list update corresponds to one window of + * classified audio, and the `timeMs` will always point to the start of that window. + * + * For an example, below is an audio stream timeline with a 1 second classifier window. + * Audio stream: + * |==========>=========|============>=========|===========>==========|===========>=========| + * 0 1000 2000 3000 4000 + * | | | | + * V V V V + * [{HUMAN, 0.8}] [{HUMAN, 0.8}, [{HUMAN, 0.8}, [{HUMAN, 0.8}] + * | {NATURE, 0.4}] {NATURE, 0.4}] | + * | | | + * V V V + * onClassifierUpdate onClassifierUpdate onClassifierUpdate + * timeMs: 0 timeMs: 1000 timeMs: 3000 + * [{HUMAN, 0.8}] [{HUMAN, 0.8}, [{HUMAN, 0.8}] + * {NATURE, 0.4}] + */ + int timeMs; + + /** + * List of classification metadata, including the sound classification, confidence score, and + * a duration since when the sound class was considered active. + * + * Metadatas in the list should be ranked in descending order based on the confidence score. + */ + ClassificationMetadata[] metadatas; +}
diff --git a/media/aidl/android/media/audio/eraser/ClassifierCapability.aidl b/media/aidl/android/media/audio/eraser/ClassifierCapability.aidl new file mode 100644 index 0000000..5348086 --- /dev/null +++ b/media/aidl/android/media/audio/eraser/ClassifierCapability.aidl
@@ -0,0 +1,48 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.audio.eraser; + +import android.media.audio.eraser.Classification; + +/** + * Represents the capabilities of a sound classifier component. + * + * This parcelable contains a list of supported sound classifications that the classifier can + * recognize and process. + */ +@JavaDerive(equals=true, toString=true) +@VintfStability +parcelable ClassifierCapability { + /** + * The window size of the classifier model in milliseconds. + * + * Indicates the duration over which the classifier processes audio data to output a + * classification result. + * + * Clients can expect to receive updates at most once per window. + */ + int windowSizeMs; + + /** + * List of supported sound classifications. + * + * Each entry specifies a sound classification category that the classifier can recognize, such + * as `HUMAN`, `MUSIC`, `ANIMAL`, etc. This defines the types of sounds the classifier is + * capable of identifying in the input audio. + */ + Classification[] supportedClassifications; +}
diff --git a/media/aidl/android/media/audio/eraser/Configuration.aidl b/media/aidl/android/media/audio/eraser/Configuration.aidl new file mode 100644 index 0000000..b693244 --- /dev/null +++ b/media/aidl/android/media/audio/eraser/Configuration.aidl
@@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.audio.eraser; + +import android.media.audio.eraser.ClassificationConfig; +import android.media.audio.eraser.IEraserCallback; +import android.media.audio.eraser.Mode; + +/** + * Eraser configurations. Configuration for eraser operation mode, sound classification behaviors, + * and an optional callback interface. + */ +@JavaDerive(equals=true, toString=true) +@VintfStability +parcelable Configuration { + /** + * Work mode for the eraser, specifies the current operating mode of the eraser effect. + */ + Mode mode = Mode.ERASER; + + /** + * List of eraser configurations. + * Each configuration defines the behavior for specific sound classifications, allowing + * different gain factors and confidence thresholds to be applied based on classification + * results. + */ + ClassificationConfig[] classificationConfigs; + + /** + * Maximum number of classification metadata generated from the sound classification. + * + * Default value set to 5. + */ + int maxClassificationMetadata = 5; + + /** + * Optional callback inerface to get the eraser effect results. + */ + @nullable IEraserCallback callback; +}
diff --git a/media/aidl/android/media/audio/eraser/IEraserCallback.aidl b/media/aidl/android/media/audio/eraser/IEraserCallback.aidl new file mode 100644 index 0000000..4863ba5 --- /dev/null +++ b/media/aidl/android/media/audio/eraser/IEraserCallback.aidl
@@ -0,0 +1,50 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.audio.eraser; + +import android.media.audio.eraser.ClassificationMetadataList; + +/** + * Callback interface for delivering results from the eraser effect. + */ +@VintfStability +interface IEraserCallback { + /** + * Provides classifier updates, including sound classifications and their confidence scores, + * along with the associated timestamp for a given `soundSourceId`. + * + * The callback is invoked when there is a change in the list of active classification metadata + * for each sound source. Changes include the addition and removal of a classification, or + * a change in the condidence score. + * + * The number of metadata elements in the `ClassificationMetadataList.metadatas` list will not + * exceed the `maxClassificationMetadata` set in `android.media.audio.eraser.Configuration`. + * + * Different classifiers may have varying window sizes, regardless of the window size, the + * classifier updates occur at most once per window per sound source. + * + * @param soundSourceId The identifier for the sound source being classified. In ERASER mode, + * this identifies the separated sound source. + * - In CLASSIFIER mode, the `soundSourceId` is always `0` as there is only one sound + * source for the eraser effect. + * - In ERASER mode, the `soundSourceId` range is [0, `maxSoundSources - 1`], where + * `maxSoundSources` is defined in the eraser capability through `SeparatorCapability`. + * + * @param metadata The classification metadata list for the current sound source. + */ + oneway void onClassifierUpdate(in int soundSourceId, in ClassificationMetadataList metadata); +}
diff --git a/media/aidl/android/media/audio/eraser/Mode.aidl b/media/aidl/android/media/audio/eraser/Mode.aidl new file mode 100644 index 0000000..30745c5 --- /dev/null +++ b/media/aidl/android/media/audio/eraser/Mode.aidl
@@ -0,0 +1,58 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.audio.eraser; + +/** + * Defines the operational mode of the Eraser effect. + */ +@JavaDerive(equals=true, toString=true) +@VintfStability +@Backing(type="byte") +enum Mode { + /** + * ERASER mode: The effect operates as an automatic sound eraser. + * + * In this mode, the Eraser effect processes the input audio using the Sound Separator, + * Sound Classifier, and Remixer components. The sound to be erased or retained is determined + * by the classifications and gain adjustments specified in eraser configuration. + * + * - The Sound Separator separates the input audio into multiple sound sources. + * - The Sound Classifier analyzes each separated sound to determine its sound category. + * - The Remixer applies gain adjustments based on the classifications and configurations, and + * re-mix the processed sounds back into a single output audio stream. + * + * Requirements: To operate in this mode, the effect must support the classifier, separator, + * and remixer capabilities. + * + * Use Cases: Selectively suppressing or enhancing specific sounds in the audio stream, + * such as removing background noise or isolating desired sound sources. + */ + ERASER, + + /** + * CLASSIFIER mode: The effect operates as a sound classifier. + * + * In this mode, the Sound Classifier analyzes the input audio in real-time and emits + * classification results based on the sound categories detected. The input audio is directly + * passed through to the output without any modification. + * + * Use Cases: Useful for applications that need to detect specific sound events, monitor audio + * content, or provide real-time visual feedback on audio classifications, without altering the + * original audio stream. + */ + CLASSIFIER, +}
diff --git a/media/aidl/android/media/audio/eraser/RemixerCapability.aidl b/media/aidl/android/media/audio/eraser/RemixerCapability.aidl new file mode 100644 index 0000000..0c89fc1 --- /dev/null +++ b/media/aidl/android/media/audio/eraser/RemixerCapability.aidl
@@ -0,0 +1,59 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.audio.eraser; + +/** + * Represents the capabilities of a sound remixer component. + * + * This parcelable defines the supported range of gainFactors that the remixer can apply to the + * audio signal. + */ +@JavaDerive(equals=true, toString=true) +@VintfStability +parcelable RemixerCapability { + /** + * Indicates whether sound remixer is supported. + * + * If `supported` is true, the sound remixer can adjust the gain of the audio signal based on + * the provided classifications and gainFactors, and remix the separated sounds into one. + */ + boolean supported; + + /** + * Minimum gainFactor supported by the sound remixer. + * + * Specifies the lowest gainFactor that the remixer can apply. A gainFactor of `0.0` + * typically mutes the sound. In some less common cases, a remixer can support a negative + * `gainFactor`, which enables some use cases like phase inversion and noise cancellation. + * + * The minimum gainFactor must be at least `0.0`. The default minimum gainFactor for a remixer + * is `0.0` (the sound is muted). + */ + float minGainFactor = 0f; + + /** + * Maximum gainFactor supported by the sound remixer. + * + * Specifies the highest gainFactor that the remixer can apply. A gainFactor of `1.0` means no + * adjustment to the sound's original volume. In the case of gainFactor greater than `1.0`, the + * remixer may apply amplification to the audio signal. + * + * The maximum gainFactor must be at least `1.0`, and the default maximum gainFactor for a + * remixer is `1.0` (no gain adjustment to the sound). + */ + float maxGainFactor = 1f; +}
diff --git a/media/aidl/android/media/audio/eraser/SeparatorCapability.aidl b/media/aidl/android/media/audio/eraser/SeparatorCapability.aidl new file mode 100644 index 0000000..93b959c --- /dev/null +++ b/media/aidl/android/media/audio/eraser/SeparatorCapability.aidl
@@ -0,0 +1,55 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.media.audio.eraser; + +/** + * Represents the capabilities of a sound separator component. + * + * This parcelable includes the maximum number of sound sources that can be separated + * simultaneously. + */ +@JavaDerive(equals=true, toString=true) +@VintfStability +parcelable SeparatorCapability { + /** + * Indicates whether sound separation is supported. + * + * Note: If sound separation is supported, the effect must also support sound remixing to + * handle the separated audio streams, and produce remixed audio as output. + */ + boolean supported; + + /** + * The minimum number of sound sources a sound separator must support. + */ + const int MIN_SOUND_SOURCE_SUPPORTED = 2; + + /** + * Maximum number of sound sources that can be separated. + * + * Specifies the maximum number of individual sound sources that the separator can process + * simultaneously. + * + * Each separated sound source have an soundSourceId, range in [0, maxSoundSources -1]. In + * ERASER mode, each sound source will be classified with a classifier, identified by the + * soundSourceId. + * + * The minimum value of `maxSoundSources` is 2 as defined by `MIN_SOUND_SOURCE_SUPPORTED`, the + * default value is 4. + */ + int maxSoundSources = 4; +}
diff --git a/media/aidl/android/media/audio/common/SoundClassification.aidl b/media/aidl/android/media/audio/eraser/SoundClassification.aidl similarity index 93% rename from media/aidl/android/media/audio/common/SoundClassification.aidl rename to media/aidl/android/media/audio/eraser/SoundClassification.aidl index 7ca15f7..6ec9b43 100644 --- a/media/aidl/android/media/audio/common/SoundClassification.aidl +++ b/media/aidl/android/media/audio/eraser/SoundClassification.aidl
@@ -14,7 +14,7 @@ * limitations under the License. */ -package android.media.audio.common; +package android.media.audio.eraser; /** * The sound classification is based on the top-level categories of the "AudioSet ontology". @@ -67,4 +67,9 @@ * foreground or target objects. */ ENVIRONMENT, + + /** + * Vendor customizable extension, for possible classifications not listed above. + */ + VENDOR_EXTENSION, }
diff --git a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/Capability.aidl similarity index 79% copy from media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl copy to media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/Capability.aidl index 156a393..a415a42 100644 --- a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl +++ b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/Capability.aidl
@@ -31,14 +31,13 @@ // with such a backward incompatible change, it has a high risk of breaking // later when a module using the interface is updated, e.g., Mainline modules. -package android.media.audio.common; -@Backing(type="int") @JavaDerive(equals=true, toString=true) @VintfStability -enum SoundClassification { - HUMAN, - ANIMAL, - NATURE, - MUSIC, - THINGS, - AMBIGUOUS, - ENVIRONMENT, +package android.media.audio.eraser; +@JavaDerive(equals=true, toString=true) @VintfStability +parcelable Capability { + int[] sampleRates; + android.media.audio.common.AudioChannelLayout[] channelLayouts; + android.media.audio.eraser.Mode[] modes; + android.media.audio.eraser.SeparatorCapability separator; + android.media.audio.eraser.ClassifierCapability classifier; + android.media.audio.eraser.RemixerCapability remixer; }
diff --git a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/Classification.aidl similarity index 87% copy from media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl copy to media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/Classification.aidl index 156a393..f90f1c1 100644 --- a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl +++ b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/Classification.aidl
@@ -31,14 +31,8 @@ // with such a backward incompatible change, it has a high risk of breaking // later when a module using the interface is updated, e.g., Mainline modules. -package android.media.audio.common; -@Backing(type="int") @JavaDerive(equals=true, toString=true) @VintfStability -enum SoundClassification { - HUMAN, - ANIMAL, - NATURE, - MUSIC, - THINGS, - AMBIGUOUS, - ENVIRONMENT, +package android.media.audio.eraser; +@JavaDerive(equals=true, toString=true) @VintfStability +parcelable Classification { + android.media.audio.eraser.SoundClassification classification = android.media.audio.eraser.SoundClassification.HUMAN; }
diff --git a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/ClassificationConfig.aidl similarity index 86% copy from media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl copy to media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/ClassificationConfig.aidl index 156a393..763352d 100644 --- a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl +++ b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/ClassificationConfig.aidl
@@ -31,14 +31,10 @@ // with such a backward incompatible change, it has a high risk of breaking // later when a module using the interface is updated, e.g., Mainline modules. -package android.media.audio.common; -@Backing(type="int") @JavaDerive(equals=true, toString=true) @VintfStability -enum SoundClassification { - HUMAN, - ANIMAL, - NATURE, - MUSIC, - THINGS, - AMBIGUOUS, - ENVIRONMENT, +package android.media.audio.eraser; +@JavaDerive(equals=true, toString=true) @VintfStability +parcelable ClassificationConfig { + android.media.audio.eraser.Classification[] classifications; + float confidenceThreshold = 0f; + float gainFactor = 1f; }
diff --git a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/ClassificationMetadata.aidl similarity index 88% copy from media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl copy to media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/ClassificationMetadata.aidl index 156a393..cfdbe5b 100644 --- a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl +++ b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/ClassificationMetadata.aidl
@@ -31,14 +31,9 @@ // with such a backward incompatible change, it has a high risk of breaking // later when a module using the interface is updated, e.g., Mainline modules. -package android.media.audio.common; -@Backing(type="int") @JavaDerive(equals=true, toString=true) @VintfStability -enum SoundClassification { - HUMAN, - ANIMAL, - NATURE, - MUSIC, - THINGS, - AMBIGUOUS, - ENVIRONMENT, +package android.media.audio.eraser; +@JavaDerive(equals=true, toString=true) @VintfStability +parcelable ClassificationMetadata { + float confidenceScore; + android.media.audio.eraser.Classification classification; }
diff --git a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/ClassificationMetadataList.aidl similarity index 88% copy from media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl copy to media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/ClassificationMetadataList.aidl index 156a393..36cef59 100644 --- a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl +++ b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/ClassificationMetadataList.aidl
@@ -31,14 +31,9 @@ // with such a backward incompatible change, it has a high risk of breaking // later when a module using the interface is updated, e.g., Mainline modules. -package android.media.audio.common; -@Backing(type="int") @JavaDerive(equals=true, toString=true) @VintfStability -enum SoundClassification { - HUMAN, - ANIMAL, - NATURE, - MUSIC, - THINGS, - AMBIGUOUS, - ENVIRONMENT, +package android.media.audio.eraser; +@JavaDerive(equals=true, toString=true) @VintfStability +parcelable ClassificationMetadataList { + int timeMs; + android.media.audio.eraser.ClassificationMetadata[] metadatas; }
diff --git a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/ClassifierCapability.aidl similarity index 87% copy from media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl copy to media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/ClassifierCapability.aidl index 156a393..fadf920 100644 --- a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl +++ b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/ClassifierCapability.aidl
@@ -31,14 +31,9 @@ // with such a backward incompatible change, it has a high risk of breaking // later when a module using the interface is updated, e.g., Mainline modules. -package android.media.audio.common; -@Backing(type="int") @JavaDerive(equals=true, toString=true) @VintfStability -enum SoundClassification { - HUMAN, - ANIMAL, - NATURE, - MUSIC, - THINGS, - AMBIGUOUS, - ENVIRONMENT, +package android.media.audio.eraser; +@JavaDerive(equals=true, toString=true) @VintfStability +parcelable ClassifierCapability { + int windowSizeMs; + android.media.audio.eraser.Classification[] supportedClassifications; }
diff --git a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/Configuration.aidl similarity index 81% copy from media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl copy to media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/Configuration.aidl index 156a393..8da4032 100644 --- a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl +++ b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/Configuration.aidl
@@ -31,14 +31,11 @@ // with such a backward incompatible change, it has a high risk of breaking // later when a module using the interface is updated, e.g., Mainline modules. -package android.media.audio.common; -@Backing(type="int") @JavaDerive(equals=true, toString=true) @VintfStability -enum SoundClassification { - HUMAN, - ANIMAL, - NATURE, - MUSIC, - THINGS, - AMBIGUOUS, - ENVIRONMENT, +package android.media.audio.eraser; +@JavaDerive(equals=true, toString=true) @VintfStability +parcelable Configuration { + android.media.audio.eraser.Mode mode = android.media.audio.eraser.Mode.ERASER; + android.media.audio.eraser.ClassificationConfig[] classificationConfigs; + int maxClassificationMetadata = 5; + @nullable android.media.audio.eraser.IEraserCallback callback; }
diff --git a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/IEraserCallback.aidl similarity index 88% copy from media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl copy to media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/IEraserCallback.aidl index 156a393..8d53405 100644 --- a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl +++ b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/IEraserCallback.aidl
@@ -31,14 +31,8 @@ // with such a backward incompatible change, it has a high risk of breaking // later when a module using the interface is updated, e.g., Mainline modules. -package android.media.audio.common; -@Backing(type="int") @JavaDerive(equals=true, toString=true) @VintfStability -enum SoundClassification { - HUMAN, - ANIMAL, - NATURE, - MUSIC, - THINGS, - AMBIGUOUS, - ENVIRONMENT, +package android.media.audio.eraser; +@VintfStability +interface IEraserCallback { + oneway void onClassifierUpdate(in int soundSourceId, in android.media.audio.eraser.ClassificationMetadataList metadata); }
diff --git a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/Mode.aidl similarity index 88% copy from media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl copy to media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/Mode.aidl index 156a393..916b314 100644 --- a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl +++ b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/Mode.aidl
@@ -31,14 +31,9 @@ // with such a backward incompatible change, it has a high risk of breaking // later when a module using the interface is updated, e.g., Mainline modules. -package android.media.audio.common; -@Backing(type="int") @JavaDerive(equals=true, toString=true) @VintfStability -enum SoundClassification { - HUMAN, - ANIMAL, - NATURE, - MUSIC, - THINGS, - AMBIGUOUS, - ENVIRONMENT, +package android.media.audio.eraser; +@Backing(type="byte") @JavaDerive(equals=true, toString=true) @VintfStability +enum Mode { + ERASER, + CLASSIFIER, }
diff --git a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/RemixerCapability.aidl similarity index 88% copy from media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl copy to media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/RemixerCapability.aidl index 156a393..82707b1 100644 --- a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl +++ b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/RemixerCapability.aidl
@@ -31,14 +31,10 @@ // with such a backward incompatible change, it has a high risk of breaking // later when a module using the interface is updated, e.g., Mainline modules. -package android.media.audio.common; -@Backing(type="int") @JavaDerive(equals=true, toString=true) @VintfStability -enum SoundClassification { - HUMAN, - ANIMAL, - NATURE, - MUSIC, - THINGS, - AMBIGUOUS, - ENVIRONMENT, +package android.media.audio.eraser; +@JavaDerive(equals=true, toString=true) @VintfStability +parcelable RemixerCapability { + boolean supported; + float minGainFactor = 0f; + float maxGainFactor = 1f; }
diff --git a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/SeparatorCapability.aidl similarity index 88% copy from media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl copy to media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/SeparatorCapability.aidl index 156a393..2e983ac 100644 --- a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl +++ b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/SeparatorCapability.aidl
@@ -31,14 +31,10 @@ // with such a backward incompatible change, it has a high risk of breaking // later when a module using the interface is updated, e.g., Mainline modules. -package android.media.audio.common; -@Backing(type="int") @JavaDerive(equals=true, toString=true) @VintfStability -enum SoundClassification { - HUMAN, - ANIMAL, - NATURE, - MUSIC, - THINGS, - AMBIGUOUS, - ENVIRONMENT, +package android.media.audio.eraser; +@JavaDerive(equals=true, toString=true) @VintfStability +parcelable SeparatorCapability { + boolean supported; + int maxSoundSources = 4; + const int MIN_SOUND_SOURCE_SUPPORTED = 2; }
diff --git a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/SoundClassification.aidl similarity index 96% rename from media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl rename to media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/SoundClassification.aidl index 156a393..e5483b4 100644 --- a/media/aidl_api/android.media.audio.common.types/current/android/media/audio/common/SoundClassification.aidl +++ b/media/aidl_api/android.media.audio.eraser.types/current/android/media/audio/eraser/SoundClassification.aidl
@@ -31,7 +31,7 @@ // with such a backward incompatible change, it has a high risk of breaking // later when a module using the interface is updated, e.g., Mainline modules. -package android.media.audio.common; +package android.media.audio.eraser; @Backing(type="int") @JavaDerive(equals=true, toString=true) @VintfStability enum SoundClassification { HUMAN, @@ -41,4 +41,5 @@ THINGS, AMBIGUOUS, ENVIRONMENT, + VENDOR_EXTENSION, }