From cd5c60e2a53d5d187ea5fecd845f7099bb11c1b1 Mon Sep 17 00:00:00 2001 From: Rocka Date: Sun, 17 May 2026 23:24:25 +0800 Subject: [PATCH] [WIP] SpeechRecognizer voice input --- app/build.gradle.kts | 1 + app/src/main/AndroidManifest.xml | 5 + .../android/input/FcitxInputMethodService.kt | 2 +- .../fcitx/fcitx5/android/input/InputView.kt | 3 + .../android/input/bar/KawaiiBarComponent.kt | 20 +- .../fcitx5/android/input/bar/ui/IdleUi.kt | 30 ++- .../input/voice/VoiceInputComponent.kt | 196 ++++++++++++++++++ gradle/libs.versions.toml | 1 + 8 files changed, 241 insertions(+), 17 deletions(-) create mode 100644 app/src/main/java/org/fcitx/fcitx5/android/input/voice/VoiceInputComponent.kt diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 53f1eb8c..ea97b66e 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -113,6 +113,7 @@ dependencies { implementation(libs.imagecropper) implementation(libs.flexbox) implementation(libs.dependency) + implementation(libs.soundwavelib) implementation(libs.timber) implementation(libs.splitties.bitflags) implementation(libs.splitties.dimensions) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index fcc089b6..874238e2 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -14,6 +14,8 @@ + + @@ -24,6 +26,9 @@ + + + ( private val horizontalCandidate: HorizontalCandidateComponent by manager.must() private val commonKeyActionListener: CommonKeyActionListener by manager.must() private val popup: PopupComponent by manager.must() + private val voiceInput: VoiceInputComponent by manager.must() private val prefs = AppPrefs.getInstance() @@ -104,8 +104,6 @@ class KawaiiBarComponent : UniqueViewComponent( private val expandedCandidateStyle by prefs.keyboard.expandedCandidateStyle private val expandToolbarByDefault by prefs.keyboard.expandToolbarByDefault private val toolbarNumRowOnPassword by prefs.keyboard.toolbarNumRowOnPassword - private val showVoiceInputButton by prefs.keyboard.showVoiceInputButton - private val preferredVoiceInput by prefs.keyboard.preferredVoiceInput private var clipboardTimeoutJob: Job? = null @@ -264,13 +262,6 @@ class KawaiiBarComponent : UniqueViewComponent( } else false } - private var voiceInputSubtype: Pair? = null - - private val switchToVoiceInputCallback = View.OnClickListener { - val (id, subtype) = voiceInputSubtype ?: return@OnClickListener - InputMethodUtil.switchInputMethod(service, id, subtype) - } - private val idleUi: IdleUi by lazy { IdleUi(context, theme, popup, commonKeyActionListener).apply { menuButton.setOnClickListener { @@ -439,6 +430,7 @@ class KawaiiBarComponent : UniqueViewComponent( ClipboardManager.addOnUpdateListener(onClipboardUpdateListener) clipboardSuggestion.registerOnChangeListener(onClipboardSuggestionUpdateListener) clipboardItemTimeout.registerOnChangeListener(onClipboardTimeoutUpdateListener) + voiceInput.addAudioVolumeListener(idleUi.audioVolumeListener) } override fun onStartInput(info: EditorInfo, capFlags: CapabilityFlags) { @@ -451,12 +443,10 @@ class KawaiiBarComponent : UniqueViewComponent( if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { idleUi.inlineSuggestionsBar.clear() } - voiceInputSubtype = InputMethodUtil.findVoiceSubtype(preferredVoiceInput) - val shouldShowVoiceInput = - showVoiceInputButton && voiceInputSubtype != null && !capFlags.has(CapabilityFlag.Password) + val shouldShowVoiceInput = voiceInput.shouldShowVoiceInput(capFlags) idleUi.setHideKeyboardIsVoiceInput( shouldShowVoiceInput, - if (shouldShowVoiceInput) switchToVoiceInputCallback else hideKeyboardCallback + if (shouldShowVoiceInput) voiceInput.voiceInputCallback else hideKeyboardCallback ) evalIdleUiState() } diff --git a/app/src/main/java/org/fcitx/fcitx5/android/input/bar/ui/IdleUi.kt b/app/src/main/java/org/fcitx/fcitx5/android/input/bar/ui/IdleUi.kt index e15bbe76..a24529d4 100644 --- a/app/src/main/java/org/fcitx/fcitx5/android/input/bar/ui/IdleUi.kt +++ b/app/src/main/java/org/fcitx/fcitx5/android/input/bar/ui/IdleUi.kt @@ -8,6 +8,7 @@ import android.content.Context import android.transition.Slide import android.transition.TransitionManager import android.transition.TransitionSet +import android.util.AttributeSet import android.view.View import android.view.Gravity import android.view.animation.AlphaAnimation @@ -15,6 +16,7 @@ import android.view.animation.AnimationSet import android.view.animation.TranslateAnimation import android.widget.Space import android.widget.ViewAnimator +import com.soundwave.lib.SoundWaveView import org.fcitx.fcitx5.android.R import org.fcitx.fcitx5.android.data.prefs.AppPrefs import org.fcitx.fcitx5.android.data.theme.Theme @@ -25,6 +27,7 @@ import org.fcitx.fcitx5.android.input.bar.ui.idle.InlineSuggestionsUi import org.fcitx.fcitx5.android.input.bar.ui.idle.NumberRow import org.fcitx.fcitx5.android.input.keyboard.CommonKeyActionListener import org.fcitx.fcitx5.android.input.popup.PopupComponent +import org.fcitx.fcitx5.android.input.voice.VoiceInputComponent import splitties.dimensions.dp import splitties.views.dsl.constraintlayout.after import splitties.views.dsl.constraintlayout.before @@ -39,8 +42,10 @@ import splitties.views.dsl.core.add import splitties.views.dsl.core.lParams import splitties.views.dsl.core.frameLayout import splitties.views.dsl.core.matchParent +import splitties.views.dsl.core.view import splitties.views.imageResource import timber.log.Timber +import kotlin.math.roundToInt class IdleUi( override val ctx: Context, @@ -77,7 +82,30 @@ class IdleUi( val hideKeyboardButton = ToolButton(ctx, R.drawable.ic_baseline_arrow_drop_down_24, theme) - val emptyBar = Space(ctx) +// val emptyBar = Space(ctx) + + // TODO: dedicated UI for active voice input + val emptyBar = SoundWaveView(ctx).apply { + enableIdle(true) + setColor(theme.altKeyTextColor) +// volumeCount = 20 +// volumeIdleCount = 8 + maxVolume = 12 + minVolume = 2 + minVolumeBarHeight = dp(3) + maxVolumeBarHeight = dp(KawaiiBarComponent.HEIGHT - 6) +// volumeBarMargin = dp(4) +// volumeBarHalfWidth = dp(2) + maxIdleHeight = maxVolumeBarHeight / 2 + } + + val audioVolumeListener = VoiceInputComponent.AudioVolumeListener { listening, dB -> + if (listening) { + emptyBar.handleVolume(dB.roundToInt()) + } else { + emptyBar.stopDance() + } + } val buttonsUi = ButtonsBarUi(ctx, theme) diff --git a/app/src/main/java/org/fcitx/fcitx5/android/input/voice/VoiceInputComponent.kt b/app/src/main/java/org/fcitx/fcitx5/android/input/voice/VoiceInputComponent.kt new file mode 100644 index 00000000..fbd87949 --- /dev/null +++ b/app/src/main/java/org/fcitx/fcitx5/android/input/voice/VoiceInputComponent.kt @@ -0,0 +1,196 @@ +/* + * SPDX-License-Identifier: LGPL-2.1-or-later + * SPDX-FileCopyrightText: Copyright 2026 Fcitx5 for Android Contributors + */ + +package org.fcitx.fcitx5.android.input.voice + +import android.content.Intent +import android.os.Build +import android.os.Bundle +import android.speech.RecognitionListener +import android.speech.RecognizerIntent +import android.speech.SpeechRecognizer +import android.view.View +import android.view.inputmethod.InputMethodSubtype +import org.fcitx.fcitx5.android.core.CapabilityFlag +import org.fcitx.fcitx5.android.core.CapabilityFlags +import org.fcitx.fcitx5.android.core.FormattedText +import org.fcitx.fcitx5.android.core.InputMethodEntry +import org.fcitx.fcitx5.android.core.TextFormatFlag +import org.fcitx.fcitx5.android.data.prefs.AppPrefs +import org.fcitx.fcitx5.android.input.broadcast.InputBroadcastReceiver +import org.fcitx.fcitx5.android.input.dependency.context +import org.fcitx.fcitx5.android.input.dependency.fcitx +import org.fcitx.fcitx5.android.input.dependency.inputMethodService +import org.fcitx.fcitx5.android.utils.InputMethodUtil +import org.fcitx.fcitx5.android.utils.WeakHashSet +import org.fcitx.fcitx5.android.utils.toast +import org.mechdancer.dependency.Dependent +import org.mechdancer.dependency.UniqueComponent +import org.mechdancer.dependency.manager.ManagedHandler +import org.mechdancer.dependency.manager.managedHandler +import timber.log.Timber + +class VoiceInputComponent : UniqueComponent(), Dependent, + ManagedHandler by managedHandler(), InputBroadcastReceiver { + + val context by manager.context() + val service by manager.inputMethodService() + val fcitx by manager.fcitx() + + private val prefs = AppPrefs.getInstance() + + private val showVoiceInputButton by prefs.keyboard.showVoiceInputButton + private val preferredVoiceInput by prefs.keyboard.preferredVoiceInput + + private var voiceInputSubtype: Pair? = null + + fun shouldShowVoiceInput(capFlags: CapabilityFlags): Boolean { + voiceInputSubtype = InputMethodUtil.findVoiceSubtype(preferredVoiceInput) + return showVoiceInputButton && voiceInputSubtype != null && !capFlags.has(CapabilityFlag.Password) + } + + // TODO: switch between "other voice input method" and "SpeechRecognizer" + val voiceInputCallback = View.OnClickListener { + val preferredIdx = InputMethodUtil.listVoiceInputMethods().indexOfFirst { (imi, subType) -> + imi.id == preferredVoiceInput + } + if (preferredIdx < 0) { + startListening() + return@OnClickListener + } + val (id, subtype) = voiceInputSubtype ?: return@OnClickListener + InputMethodUtil.switchInputMethod(service, id, subtype) + } + + private var languageCode = "" + + override fun onImeUpdate(ime: InputMethodEntry) { + languageCode = ime.languageCode.replace("_", "-") + } + + fun Bundle.results() = getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION) + + fun Bundle.dumpResults() = buildString { + val results = getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION) + val scores = getFloatArray(SpeechRecognizer.CONFIDENCE_SCORES) + results?.forEachIndexed { index, string -> + append(string) + append("[score=${scores?.get(index)}]") + if (index > 0) append(", ") + } + } + + // TODO: destroy recognizer onFinishInput + private val speechRecognizer: SpeechRecognizer by lazy { + SpeechRecognizer.createSpeechRecognizer(context).apply { + setRecognitionListener(recognitionListener) + } + } + + private var startedListening = false + + private fun buildUnderlineText(str: String): FormattedText { + return FormattedText(arrayOf(str), intArrayOf(TextFormatFlag.Underline.flag), -1) + } + + fun interface AudioVolumeListener { + fun onAudioVolumeChange(listening: Boolean, dB: Float) + } + + private val audioVolumeListeners = WeakHashSet() + + fun addAudioVolumeListener(listener: AudioVolumeListener) { + audioVolumeListeners.add(listener) + } + + fun removeAudioVolumeListener(listener: AudioVolumeListener) { + audioVolumeListeners.remove(listener) + } + + // TODO: interrupt voice input on keyboard input + private val recognitionListener by lazy { + object : RecognitionListener { + override fun onReadyForSpeech(params: Bundle) { + startedListening = true + audioVolumeListeners.forEach { it.onAudioVolumeChange(true, 0f) } + Timber.d("onReadyForSpeech, $params") + } + + override fun onBeginningOfSpeech() { + Timber.d("onBeginningOfSpeech") + } + + override fun onRmsChanged(rmsdB: Float) { + Timber.d("onRmsChanged: rmsdB=$rmsdB") + audioVolumeListeners.forEach { it.onAudioVolumeChange(true, rmsdB) } + } + + override fun onBufferReceived(buffer: ByteArray) { + /* This would never be called */ + } + + override fun onEndOfSpeech() { + Timber.d("onEndOfSpeech") + } + + override fun onError(error: Int) { + startedListening = false + audioVolumeListeners.forEach { it.onAudioVolumeChange(false, 0f) } + Timber.d("onError: $error") + context.toast("onError: $error") + when (error) { + SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS -> { + // TODO: launch activity to request permission + } + SpeechRecognizer.ERROR_CLIENT -> { + } + else -> { + } + } + } + + override fun onPartialResults(partialResults: Bundle) { + Timber.d("onPartialResults: ${partialResults.dumpResults()}") + val strings = partialResults.results() ?: return + // TODO: don't call IMS directly + service.updateComposingText(buildUnderlineText(strings[0])) + } + + override fun onResults(results: Bundle) { + startedListening = false + audioVolumeListeners.forEach { it.onAudioVolumeChange(false, 0f) } + Timber.d("onResults: ${results.dumpResults()}") + val strings = results.results() ?: return + service.commitText(strings[0]) + } + + override fun onEvent(eventType: Int, params: Bundle) { + /* unused */ + } + } + } + + fun startListening() { + if (startedListening) { + startedListening = false + speechRecognizer.stopListening() + return + } + + speechRecognizer.startListening(Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + // required + putExtra( + RecognizerIntent.EXTRA_LANGUAGE_MODEL, + RecognizerIntent.LANGUAGE_MODEL_FREE_FORM + ) + // optional + if (languageCode.isNotBlank()) { + putExtra(RecognizerIntent.EXTRA_LANGUAGE, languageCode) + } + putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true) + putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1) + }) + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 639e8d34..37d801a8 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -44,6 +44,7 @@ arrow-functions = { module = "io.arrow-kt:arrow-functions", version.ref = "arrow imagecropper = { module = "com.vanniktech:android-image-cropper", version = "4.7.0" } flexbox = { module = "com.google.android.flexbox:flexbox", version = "3.0.0" } dependency = { module = "org.mechdancer:dependency", version = "0.1.2" } +soundwavelib = { module = "io.github.ultimatehandsomeboy666:soundwavelib", version = "1.0.1" } timber = { module = "com.jakewharton.timber:timber", version = "5.0.1" } splitties-bitflags = { module = "com.louiscad.splitties:splitties-bitflags", version.ref = "splitties" } splitties-dimensions = { module = "com.louiscad.splitties:splitties-dimensions", version.ref = "splitties" }