nas-tools/dashboard/frontend/src/lib/voice.js

const LANGS = ["en-US", "zh-CN", "zh-HK"];
const LANG_LABELS = ["EN", "普", "粤"];
const TALK_MODES = ["auto", "tap", "hold"];
const MODE_LABELS = { auto: "Auto", tap: "Tap", hold: "Hold" };
const NOISE = /^[\s│┃┆┇┊┋─━┄┅┈┉═╌╍╔╗╚╝╠╣╦╩╬├┤┬┴┼╭╮╯╰░▒▓█▌▐▀▄⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏◐◑◒◓⣾⣽⣻⢿⡿⣟⣯⣷⠁⠂⠄⡀⢀⠠⠐⠈\-–—_=+*#~<>/.\\|,;:!?(){}\[\]]+$/;
const SKIP = /shortcuts|\/help|\/exit|press esc|ctrl[+-]|⌘|⎋|❯|^\$\s|^root@|^claude|^jimmy|^\[.*\]$/i;
const MODE_COMMANDS = {
  "hold mode": "hold", "tap mode": "tap", "auto mode": "auto",
  "按住模式": "hold", "点击模式": "tap", "自动模式": "auto",
};

export class VoiceSession {
  constructor(onChange) {
    this.onChange = onChange;
    this.ttsEnabled = false;
    this.speaking = false;
    this.listening = false;
    this.sttAvailable = !!(window.SpeechRecognition || window.webkitSpeechRecognition);
    this.langIdx = 0;
    this.voiceMode = false;
    this.transcript = "";
    this.talkMode = "auto";
    this.voices = [];
    this.voiceIdx = -1;
    this.term = null;
    this.ws = null;
    this._lastLine = 0;
    this._debounce = null;
    this._recog = null;
    this._onWriteDispose = null;
    this._ttsUnlocked = false;
    this._silenceTimer = null;
  }

  get lang() { return LANGS[this.langIdx]; }
  get langLabel() { return LANG_LABELS[this.langIdx]; }
  get modeLabel() { return MODE_LABELS[this.talkMode]; }
  get voiceName() { return this.voices[this.voiceIdx]?.name || "Default"; }

  loadVoices() {
    const all = speechSynthesis.getVoices();
    const prefix = this.lang.split("-")[0];
    this.voices = all.filter(v => v.lang.startsWith(prefix));
    // Try to keep current selection
    if (this.voiceIdx >= this.voices.length) this.voiceIdx = 0;
    if (!this.voices.length) this.voiceIdx = -1;
    this._notify();
  }

  setVoice(idx) {
    this.voiceIdx = idx;
    this._notify();
  }

  attach(term, ws) {
    this.term = term;
    this.ws = ws;
    this._lastLine = term.buffer.active.cursorY + term.buffer.active.baseY;
    this._onWriteDispose = term.onWriteParsed(() => this._onOutput());
  }

  _notify() { this.onChange?.(); }

  // --- Voice Mode ---
  enterVoiceMode() {
    this.voiceMode = true;
    this.ttsEnabled = true;
    if (!this._ttsUnlocked) {
      speechSynthesis.speak(new SpeechSynthesisUtterance(""));
      this._ttsUnlocked = true;
    }
    this.loadVoices();
    speechSynthesis.onvoiceschanged = () => this.loadVoices();
    if (this.term) {
      this._lastLine = this.term.buffer.active.baseY + this.term.buffer.active.cursorY;
    }
    if (this.talkMode === "auto") this._startSTT(true);
    this._notify();
  }

  exitVoiceMode() {
    this.voiceMode = false;
    this.ttsEnabled = false;
    this.transcript = "";
    speechSynthesis.cancel();
    this.speaking = false;
    this._stopSTT();
    this._notify();
  }

  setTalkMode(mode) {
    this.talkMode = mode;
    this._stopSTT();
    this.transcript = "";
    if (mode === "auto" && this.voiceMode) this._startSTT(true);
    this._notify();
  }

  cycleTalkMode() {
    const i = (TALK_MODES.indexOf(this.talkMode) + 1) % TALK_MODES.length;
    this.setTalkMode(TALK_MODES[i]);
  }

  // Hold mode
  startHold() {
    if (this.talkMode !== "hold") return;
    this._startSTT(false);
  }

  releaseHold() {
    if (this.talkMode !== "hold") return;
    this._sendAndStop();
  }

  // Tap mode
  tapMic() {
    if (this.talkMode !== "tap") return;
    if (this.listening) { this._sendAndStop(); }
    else { this._startSTT(false); }
  }

  _sendAndStop() {
    const text = this.transcript.trim();
    this._stopSTT();
    if (text && this.ws?.readyState === 1) this._typeText(text + "\r");
    this.transcript = "";
    this._notify();
  }

  // --- STT ---
  _startSTT(continuous) {
    const SR = window.SpeechRecognition || window.webkitSpeechRecognition;
    if (!SR) return;
    speechSynthesis.cancel();
    this.speaking = false;
    this._recog = new SR();
    this._recog.continuous = continuous;
    this._recog.interimResults = true;
    this._recog.lang = this.lang;
    this._finalText = "";
    this._recog.onresult = (e) => {
      let interim = "";
      for (let i = e.resultIndex; i < e.results.length; i++) {
        const t = e.results[i][0].transcript;
        if (e.results[i].isFinal) {
          // Check voice commands
          const cmd = MODE_COMMANDS[t.trim().toLowerCase()];
          if (cmd) { this.setTalkMode(cmd); return; }
          this._finalText += t;
        } else { interim = t; }
      }
      this.transcript = (this._finalText + interim).trim();
      this._notify();
      // Auto mode: reset silence timer on new results
      if (this.talkMode === "auto" && this._finalText) {
        clearTimeout(this._silenceTimer);
        this._silenceTimer = setTimeout(() => this._autoSend(), 2000);
      }
    };
    this._recog.onerror = () => { this.listening = false; this._notify(); };
    this._recog.onend = () => {
      if (this.listening && continuous) {
        // Auto mode: send accumulated text, restart
        if (this.talkMode === "auto" && this._finalText.trim()) this._autoSend();
        try { this._recog.start(); } catch { this.listening = false; this._notify(); }
      } else { this.listening = false; this._notify(); }
    };
    this._recog.start();
    this.listening = true;
    this._notify();
  }

  _stopSTT() {
    this.listening = false;
    clearTimeout(this._silenceTimer);
    this._recog?.stop();
    this._recog = null;
    this._finalText = "";
  }

  _autoSend() {
    clearTimeout(this._silenceTimer);
    const text = this._finalText.trim();
    this._finalText = "";
    this.transcript = "";
    if (text && this.ws?.readyState === 1) this._typeText(text + "\r");
    this._notify();
  }

  // --- TTS ---
  _onOutput() {
    if (!this.ttsEnabled || this.transcript) return;
    clearTimeout(this._debounce);
    this._debounce = setTimeout(() => this._readBuffer(), 2000);
  }

  _readBuffer() {
    const buf = this.term?.buffer?.active;
    if (!buf || buf.type !== "normal") return;
    const curLine = buf.baseY + buf.cursorY;
    const lines = [];
    for (let y = this._lastLine + 1; y <= curLine; y++) {
      const row = buf.getLine(y);
      if (!row) continue;
      const text = row.translateToString(true).trim();
      if (text.length >= 4 && !NOISE.test(text) && !SKIP.test(text)) lines.push(text);
    }
    this._lastLine = curLine;
    if (lines.length) this._speak(lines.join(". "));
  }

  _speak(text) {
    if (!text) return;
    speechSynthesis.cancel();
    const u = new SpeechSynthesisUtterance(text);
    u.lang = this.lang;
    if (this.voices[this.voiceIdx]) u.voice = this.voices[this.voiceIdx];
    u.onstart = () => { this.speaking = true; this._notify(); };
    u.onend = () => { this.speaking = false; this._notify(); };
    u.onerror = () => { this.speaking = false; this._notify(); };
    speechSynthesis.speak(u);
  }

  _typeText(text) {
    let i = 0;
    const next = () => {
      if (i < text.length && this.ws?.readyState === 1) {
        this.ws.send(new TextEncoder().encode(text[i++]));
        setTimeout(next, 20);
      }
    };
    next();
  }

  cycleLang() {
    this.langIdx = (this.langIdx + 1) % LANGS.length;
    this.loadVoices();
    if (this.listening) {
      const wasContinuous = this._recog?.continuous;
      this._stopSTT();
      this._startSTT(!!wasContinuous);
    }
    this._notify();
  }

  destroy() {
    clearTimeout(this._debounce);
    clearTimeout(this._silenceTimer);
    this._onWriteDispose?.dispose();
    this._stopSTT();
    speechSynthesis.cancel();
    this.term = null;
    this.ws = null;
  }
}