From 70c716fd191564a613310fb4156e312b872471c9 Mon Sep 17 00:00:00 2001 From: "Gan, Jimmy" Date: Thu, 26 Feb 2026 16:03:43 +0800 Subject: [PATCH] feat: voice mode bottom sheet with hold/tap/auto talk modes --- dashboard/frontend/src/lib/voice.js | 205 ++++++++++++------ dashboard/frontend/src/routes/Terminal.svelte | 99 +++++++-- 2 files changed, 218 insertions(+), 86 deletions(-) diff --git a/dashboard/frontend/src/lib/voice.js b/dashboard/frontend/src/lib/voice.js index db7e9ac..3273681 100644 --- a/dashboard/frontend/src/lib/voice.js +++ b/dashboard/frontend/src/lib/voice.js @@ -1,6 +1,12 @@ const LANGS = ["en-US", "zh-CN", "zh-HK"]; const LANG_LABELS = ["EN", "普", "粤"]; +const TALK_MODES = ["auto", "tap", "hold"]; +const MODE_LABELS = { auto: "Auto", tap: "Tap", hold: "Hold" }; const NOISE = /^[\s│┃┆┇┊┋─━┄┅┈┉═╌╍╔╗╚╝╠╣╦╩╬├┤┬┴┼╭╮╯╰░▒▓█▌▐▀▄⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏◐◑◒◓⣾⣽⣻⢿⡿⣟⣯⣷⠁⠂⠄⡀⢀⠠⠐⠈\-–—_=+*#~<>/.\\|,;:!?(){}\[\]]+$/; +const MODE_COMMANDS = { + "hold mode": "hold", "tap mode": "tap", "auto mode": "auto", + "按住模式": "hold", "点击模式": "tap", "自动模式": "auto", +}; export class VoiceSession { constructor(onChange) { @@ -10,6 +16,9 @@ export class VoiceSession { this.listening = false; this.sttAvailable = !!(window.SpeechRecognition || window.webkitSpeechRecognition); this.langIdx = 0; + this.voiceMode = false; + this.transcript = ""; + this.talkMode = "auto"; this.term = null; this.ws = null; this._lastLine = 0; @@ -17,10 +26,12 @@ export class VoiceSession { this._recog = null; this._onWriteDispose = null; this._ttsUnlocked = false; + this._silenceTimer = null; } get lang() { return LANGS[this.langIdx]; } get langLabel() { return LANG_LABELS[this.langIdx]; } + get modeLabel() { return MODE_LABELS[this.talkMode]; } attach(term, ws) { this.term = term; @@ -31,6 +42,131 @@ export class VoiceSession { _notify() { this.onChange?.(); } + // --- Voice Mode --- + enterVoiceMode() { + this.voiceMode = true; + this.ttsEnabled = true; + if (!this._ttsUnlocked) { + speechSynthesis.speak(new SpeechSynthesisUtterance("")); + this._ttsUnlocked = true; + } + if (this.term) { + this._lastLine = this.term.buffer.active.baseY + this.term.buffer.active.cursorY; + } + if (this.talkMode === "auto") this._startSTT(true); + this._notify(); + } + + exitVoiceMode() { + this.voiceMode = false; + this.ttsEnabled = false; + this.transcript = ""; + speechSynthesis.cancel(); + this.speaking = false; + this._stopSTT(); + this._notify(); + } + + setTalkMode(mode) { + this.talkMode = mode; + this._stopSTT(); + this.transcript = ""; + if (mode === "auto" && this.voiceMode) this._startSTT(true); + this._notify(); + } + + cycleTalkMode() { + const i = (TALK_MODES.indexOf(this.talkMode) + 1) % TALK_MODES.length; + this.setTalkMode(TALK_MODES[i]); + } + + // Hold mode + startHold() { + if (this.talkMode !== "hold") return; + this._startSTT(false); + } + + releaseHold() { + if (this.talkMode !== "hold") return; + this._sendAndStop(); + } + + // Tap mode + tapMic() { + if (this.talkMode !== "tap") return; + if (this.listening) { this._sendAndStop(); } + else { this._startSTT(false); } + } + + _sendAndStop() { + const text = this.transcript.trim(); + this._stopSTT(); + if (text && this.ws?.readyState === 1) this._typeText(text + "\r"); + this.transcript = ""; + this._notify(); + } + + // --- STT --- + _startSTT(continuous) { + const SR = window.SpeechRecognition || window.webkitSpeechRecognition; + if (!SR) return; + speechSynthesis.cancel(); + this.speaking = false; + this._recog = new SR(); + this._recog.continuous = continuous; + this._recog.interimResults = true; + this._recog.lang = this.lang; + this._finalText = ""; + this._recog.onresult = (e) => { + let interim = ""; + for (let i = e.resultIndex; i < e.results.length; i++) { + const t = e.results[i][0].transcript; + if (e.results[i].isFinal) { + // Check voice commands + const cmd = MODE_COMMANDS[t.trim().toLowerCase()]; + if (cmd) { this.setTalkMode(cmd); return; } + this._finalText += t; + } else { interim = t; } + } + this.transcript = (this._finalText + interim).trim(); + this._notify(); + // Auto mode: reset silence timer on new results + if (this.talkMode === "auto" && this._finalText) { + clearTimeout(this._silenceTimer); + this._silenceTimer = setTimeout(() => this._autoSend(), 2000); + } + }; + this._recog.onerror = () => { this.listening = false; this._notify(); }; + this._recog.onend = () => { + if (this.listening && continuous) { + // Auto mode: send accumulated text, restart + if (this.talkMode === "auto" && this._finalText.trim()) this._autoSend(); + try { this._recog.start(); } catch { this.listening = false; this._notify(); } + } else { this.listening = false; this._notify(); } + }; + this._recog.start(); + this.listening = true; + this._notify(); + } + + _stopSTT() { + this.listening = false; + clearTimeout(this._silenceTimer); + this._recog?.stop(); + this._recog = null; + this._finalText = ""; + } + + _autoSend() { + clearTimeout(this._silenceTimer); + const text = this._finalText.trim(); + this._finalText = ""; + this.transcript = ""; + if (text && this.ws?.readyState === 1) this._typeText(text + "\r"); + this._notify(); + } + + // --- TTS --- _onOutput() { if (!this.ttsEnabled || this.listening) return; clearTimeout(this._debounce); @@ -66,24 +202,6 @@ export class VoiceSession { speechSynthesis.speak(u); } - toggleTts() { - this.ttsEnabled = !this.ttsEnabled; - if (!this.ttsEnabled) { speechSynthesis.cancel(); this.speaking = false; } - if (this.ttsEnabled) { - // iOS Safari requires first speak() in a user gesture to unlock audio - if (!this._ttsUnlocked) { - const u = new SpeechSynthesisUtterance(""); - speechSynthesis.speak(u); - this._ttsUnlocked = true; - } - if (this.term) { - const buf = this.term.buffer.active; - this._lastLine = buf.baseY + buf.cursorY; - } - } - this._notify(); - } - _typeText(text) { let i = 0; const next = () => { @@ -95,60 +213,23 @@ export class VoiceSession { next(); } - toggleMic() { - this.listening ? this.stopListening() : this.startListening(); - } - - startListening() { - const SR = window.SpeechRecognition || window.webkitSpeechRecognition; - if (!SR) return; - speechSynthesis.cancel(); - this.speaking = false; - this._recog = new SR(); - this._recog.continuous = true; - this._recog.interimResults = false; - this._recog.lang = this.lang; - this._recog.onresult = (e) => { - for (let i = e.resultIndex; i < e.results.length; i++) { - if (e.results[i].isFinal) { - const text = e.results[i][0].transcript.trim(); - if (text && this.ws?.readyState === 1) this._typeText(text + "\r"); - } - } - }; - this._recog.onerror = () => { this.listening = false; this._notify(); }; - this._recog.onend = () => { - if (this.listening) { - try { this._recog.start(); } catch { this.listening = false; this._notify(); } - } - }; - this._recog.start(); - this.listening = true; - this._notify(); - } - - stopListening() { - this.listening = false; - this._recog?.stop(); - this._recog = null; - this._notify(); - } - cycleLang() { this.langIdx = (this.langIdx + 1) % LANGS.length; - if (this._recog && this.listening) { - this.stopListening(); - this.startListening(); + if (this.listening) { + const wasContinuous = this._recog?.continuous; + this._stopSTT(); + this._startSTT(!!wasContinuous); } this._notify(); } destroy() { clearTimeout(this._debounce); + clearTimeout(this._silenceTimer); this._onWriteDispose?.dispose(); - this.stopListening(); + this._stopSTT(); speechSynthesis.cancel(); this.term = null; this.ws = null; } -} +} \ No newline at end of file diff --git a/dashboard/frontend/src/routes/Terminal.svelte b/dashboard/frontend/src/routes/Terminal.svelte index f791dda..df66374 100644 --- a/dashboard/frontend/src/routes/Terminal.svelte +++ b/dashboard/frontend/src/routes/Terminal.svelte @@ -144,33 +144,15 @@ {#if activeTab && tabs.find(t => t.id === activeTab)?.connected && activeVoice()} {@const v = activeVoice()} -
+
- -
{/if}
@@ -185,9 +167,78 @@
{/each} + + {#if activeVoice()?.voiceMode} + {@const v = activeVoice()} +
+ +
+ Voice Mode +
+ + + +
+
+ +
+ {#if v.transcript} +

"{v.transcript}"

+ {:else} +

{v.speaking ? 'Speaking...' : 'Waiting for speech...'}

+ {/if} +
+ +
+ {#if v.talkMode === "hold"} + + {v.listening ? 'Release to send' : 'Hold to talk'} + {:else if v.talkMode === "tap"} + + {v.listening ? 'Tap to send' : 'Tap to talk'} + {:else} +
+ +
+ {v.listening ? 'Listening...' : v.speaking ? 'Speaking...' : 'Ready'} + {/if} +
+
+ {/if} + +