From 9c5dbcdb3717939dcd8bda4c24eb5a409a524a61 Mon Sep 17 00:00:00 2001 From: "Gan, Jimmy" Date: Thu, 26 Feb 2026 15:35:43 +0800 Subject: [PATCH] feat: add voice mode (STT/TTS) to terminal page --- dashboard/frontend/src/lib/voice.js | 135 ++++++++++++++++++ dashboard/frontend/src/routes/Terminal.svelte | 43 +++++- 2 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 dashboard/frontend/src/lib/voice.js diff --git a/dashboard/frontend/src/lib/voice.js b/dashboard/frontend/src/lib/voice.js new file mode 100644 index 0000000..2940f93 --- /dev/null +++ b/dashboard/frontend/src/lib/voice.js @@ -0,0 +1,135 @@ +const LANGS = ["en-US", "zh-CN", "zh-HK"]; +const LANG_LABELS = ["EN", "普", "粤"]; +const NOISE = /^[\s│┃┆┇┊┋─━┄┅┈┉═╌╍╔╗╚╝╠╣╦╩╬├┤┬┴┼╭╮╯╰░▒▓█▌▐▀▄⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏◐◑◒◓⣾⣽⣻⢿⡿⣟⣯⣷⠁⠂⠄⡀⢀⠠⠐⠈\-–—_=+*#~<>/.\\|,;:!?(){}\[\]]+$/; + +export class VoiceSession { + constructor(onChange) { + this.onChange = onChange; + this.ttsEnabled = false; + this.speaking = false; + this.listening = false; + this.langIdx = 0; + this.term = null; + this.ws = null; + this._lastLine = 0; + this._debounce = null; + this._recog = null; + this._onWriteDispose = null; + } + + get lang() { return LANGS[this.langIdx]; } + get langLabel() { return LANG_LABELS[this.langIdx]; } + + attach(term, ws) { + this.term = term; + this.ws = ws; + this._lastLine = term.buffer.active.cursorY + term.buffer.active.baseY; + this._onWriteDispose = term.onWriteParsed(() => this._onOutput()); + } + + _notify() { this.onChange?.(); } + + _onOutput() { + if (!this.ttsEnabled || this.listening) return; + clearTimeout(this._debounce); + this._debounce = setTimeout(() => this._readBuffer(), 600); + } + + _readBuffer() { + const buf = this.term?.buffer?.active; + if (!buf || buf.type !== "normal") return; + const curLine = buf.baseY + buf.cursorY; + const lines = []; + for (let y = this._lastLine + 1; y <= curLine; y++) { + const row = buf.getLine(y); + if (!row) continue; + const text = row.translateToString(true).trim(); + if (text.length >= 4 && !NOISE.test(text)) lines.push(text); + } + this._lastLine = curLine; + if (lines.length) this._speak(lines.join(". ")); + } + + _speak(text) { + if (!text) return; + speechSynthesis.cancel(); + const u = new SpeechSynthesisUtterance(text); + u.lang = this.lang; + const voices = speechSynthesis.getVoices(); + const v = voices.find(v => v.lang.startsWith(this.lang.split("-")[0])); + if (v) u.voice = v; + u.onstart = () => { this.speaking = true; this._notify(); }; + u.onend = () => { this.speaking = false; this._notify(); }; + u.onerror = () => { this.speaking = false; this._notify(); }; + speechSynthesis.speak(u); + } + + toggleTts() { + this.ttsEnabled = !this.ttsEnabled; + if (!this.ttsEnabled) { speechSynthesis.cancel(); this.speaking = false; } + if (this.ttsEnabled && this.term) { + const buf = this.term.buffer.active; + this._lastLine = buf.baseY + buf.cursorY; + } + this._notify(); + } + + toggleMic() { + this.listening ? this.stopListening() : this.startListening(); + } + + startListening() { + const SR = window.SpeechRecognition || window.webkitSpeechRecognition; + if (!SR) return; + speechSynthesis.cancel(); + this.speaking = false; + this._recog = new SR(); + this._recog.continuous = true; + this._recog.interimResults = false; + this._recog.lang = this.lang; + this._recog.onresult = (e) => { + for (let i = e.resultIndex; i < e.results.length; i++) { + if (e.results[i].isFinal) { + const text = e.results[i][0].transcript.trim(); + if (text && this.ws?.readyState === 1) { + this.ws.send(new TextEncoder().encode(text + "\n")); + } + } + } + }; + this._recog.onerror = () => { this.listening = false; this._notify(); }; + this._recog.onend = () => { + if (this.listening) { + try { this._recog.start(); } catch { this.listening = false; this._notify(); } + } + }; + this._recog.start(); + this.listening = true; + this._notify(); + } + + stopListening() { + this.listening = false; + this._recog?.stop(); + this._recog = null; + this._notify(); + } + + cycleLang() { + this.langIdx = (this.langIdx + 1) % LANGS.length; + if (this._recog && this.listening) { + this.stopListening(); + this.startListening(); + } + this._notify(); + } + + destroy() { + clearTimeout(this._debounce); + this._onWriteDispose?.dispose(); + this.stopListening(); + speechSynthesis.cancel(); + this.term = null; + this.ws = null; + } +} diff --git a/dashboard/frontend/src/routes/Terminal.svelte b/dashboard/frontend/src/routes/Terminal.svelte index 9958db5..dbd5743 100644 --- a/dashboard/frontend/src/routes/Terminal.svelte +++ b/dashboard/frontend/src/routes/Terminal.svelte @@ -1,5 +1,6 @@

Terminal

@@ -135,6 +142,38 @@
{/if} + {#if activeTab && tabs.find(t => t.id === activeTab)?.connected} + {@const v = activeVoice()} + {#if v} +
+ + + +
+ {/if} + {/if} {#if !tabs.length}