mirror of
https://github.com/wahyd4/links.git
synced 2026-08-09 05:06:16 +10:00
junvandCopilot
d99a362a37
- Use a thinking/running face while speech audio is being generated - Switch to speaking only when playback actually begins - Add quota-safe history persistence that strips audio from older entries instead of crashing localStorage writes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
514 lines
20 KiB
React
514 lines
20 KiB
React
import { useState, useEffect, useRef, useCallback } from 'react';
|
||
import { useNavigate } from 'react-router-dom';
|
||
import RobotFace from './components/RobotFace';
|
||
import ChatPanel from './components/ChatPanel';
|
||
import MemoryPanel from './components/MemoryPanel';
|
||
import { EXPRESSIONS, ALL_AUTO_IDS, byId } from './data/expressions';
|
||
import { apiChat, apiTts, apiAsr, apiExtractMemory, apiConfig } from './services/api';
|
||
import { usePushToTalk } from './hooks/usePushToTalk';
|
||
import { addEntry } from './store/historyStore';
|
||
import { getMemories, addMemory } from './store/memoryStore';
|
||
import { getSettings, getEffectiveVoice } from './store/settingsStore';
|
||
import { VOICES, getVoiceLabel } from './data/voices';
|
||
import './App.css';
|
||
|
||
const INTERACTION = EXPRESSIONS.filter(e => e.category === 'interaction');
|
||
const AUTO = EXPRESSIONS.filter(e => e.category === 'auto');
|
||
|
||
const KEY_MAP = {
|
||
'1': 'idle', '2': 'listening', '3': 'thinking', '4': 'speaking',
|
||
'5': 'happy', '6': 'error', '7': 'confused', '8': 'dreaming',
|
||
'q': 'natural_smile', 'w': 'calm_relax', 'e': 'curious_observe', 'r': 'light_joy',
|
||
't': 'side_think', 'y': 'warm_friendly', 'u': 'quiet_listen', 'i': 'focus_gaze',
|
||
};
|
||
|
||
// Play base64-encoded audio blob; resolves when playback ends or on error
|
||
async function playAudioBase64(base64, mimeType = 'audio/mp3') {
|
||
return new Promise((resolve) => {
|
||
const audio = new Audio(`data:${mimeType};base64,${base64}`);
|
||
audio.onended = resolve;
|
||
audio.onerror = resolve; // don't block on error
|
||
audio.play().catch(resolve);
|
||
});
|
||
}
|
||
|
||
// Browser TTS fallback
|
||
function speakText(text) {
|
||
return new Promise((resolve) => {
|
||
if (!window.speechSynthesis) return resolve();
|
||
const utt = new SpeechSynthesisUtterance(text);
|
||
utt.lang = 'zh-CN';
|
||
utt.rate = 1.05;
|
||
utt.onend = resolve;
|
||
utt.onerror = resolve;
|
||
window.speechSynthesis.speak(utt);
|
||
});
|
||
}
|
||
|
||
function stripMarkdownForSpeech(text) {
|
||
if (!text) return '';
|
||
return text
|
||
.replace(/```(?:[a-zA-Z0-9_-]+)?\n([\s\S]*?)```/g, (_, code) => code)
|
||
.replace(/`([^`]+)`/g, '$1')
|
||
.replace(/!\[([^\]]*)\]\([^)]+\)/g, '$1')
|
||
.replace(/\[([^\]]+)\]\(([^)]+)\)/g, '$1')
|
||
.replace(/^\s{0,3}#{1,6}\s+/gm, '')
|
||
.replace(/^\s{0,3}>\s?/gm, '')
|
||
.replace(/^\s*[-*+]\s+/gm, '• ')
|
||
.replace(/^\s*\d+\.\s+/gm, '')
|
||
.replace(/(\*\*|__)(.*?)\1/g, '$2')
|
||
.replace(/(\*|_)(.*?)\1/g, '$2')
|
||
.replace(/\|/g, ' ')
|
||
.replace(/\r/g, '')
|
||
.replace(/[ \t]+\n/g, '\n')
|
||
.replace(/\n{3,}/g, '\n\n')
|
||
.trim();
|
||
}
|
||
|
||
// ── Hold-to-talk voice button (defined outside App to keep stable identity) ──
|
||
function VoiceButton({ phase, onStart, onEnd, onCancel }) {
|
||
const busy = phase === 'thinking' || phase === 'speaking';
|
||
// Show cancel any time we're not fully idle
|
||
const showCancel = phase !== 'idle';
|
||
let label = '🎙 按住说话';
|
||
let mod = '';
|
||
if (phase === 'acquiring') { label = '⏳ 准备中…'; mod = ' vbtn--acquiring'; }
|
||
else if (phase === 'listening') { label = '🔴 松开发送'; mod = ' vbtn--recording'; }
|
||
else if (phase === 'thinking') { label = '⚙ 思考中…'; mod = ' vbtn--busy'; }
|
||
else if (phase === 'speaking') { label = '🔊 播放中…'; mod = ' vbtn--busy'; }
|
||
|
||
const startEvt = (e) => { e.preventDefault(); if (!busy) onStart(); };
|
||
// Button-level end — global window listener is the primary fallback
|
||
const endEvt = (e) => { e.preventDefault(); onEnd(); };
|
||
|
||
return (
|
||
<div className="vbtn-wrap">
|
||
<button
|
||
className={`vbtn${mod}`}
|
||
disabled={busy}
|
||
onMouseDown={startEvt}
|
||
onMouseUp={endEvt}
|
||
onMouseLeave={endEvt}
|
||
onTouchStart={startEvt}
|
||
onTouchEnd={endEvt}
|
||
onTouchCancel={endEvt}
|
||
>
|
||
{label}
|
||
</button>
|
||
{showCancel && (
|
||
<button className="vbtn-cancel" onClick={onCancel} title="取消">✕</button>
|
||
)}
|
||
</div>
|
||
);
|
||
}
|
||
|
||
export default function App() {
|
||
const navigate = useNavigate();
|
||
// ── Expression / auto-rotate state ───────────────────────────
|
||
const [exprId, setExprId] = useState(ALL_AUTO_IDS[0]);
|
||
const [autoRotate, setAutoRotate] = useState(true);
|
||
const [scale, setScale] = useState(1);
|
||
// switchMs is read from settings on mount (changed only via Settings page)
|
||
const [switchMs] = useState(() => getSettings().switchIntervalMs ?? 10000);
|
||
|
||
const autoTimerRef = useRef(null);
|
||
const autoRotateRef = useRef(true);
|
||
const exprIdRef = useRef(ALL_AUTO_IDS[0]);
|
||
const scheduleRef = useRef(null);
|
||
const switchMsRef = useRef(switchMs);
|
||
|
||
useEffect(() => { autoRotateRef.current = autoRotate; }, [autoRotate]);
|
||
useEffect(() => { exprIdRef.current = exprId; }, [exprId]);
|
||
|
||
const clearAuto = useCallback(() => {
|
||
clearTimeout(autoTimerRef.current);
|
||
autoTimerRef.current = null;
|
||
}, []);
|
||
|
||
scheduleRef.current = () => {
|
||
clearAuto();
|
||
if (!autoRotateRef.current) return;
|
||
const jitter = 1 + (Math.random() - 0.5) * 0.3;
|
||
autoTimerRef.current = setTimeout(() => {
|
||
const candidates = ALL_AUTO_IDS.filter(id => id !== exprIdRef.current);
|
||
const nextId = candidates[Math.floor(Math.random() * candidates.length)];
|
||
setExprId(nextId);
|
||
scheduleRef.current?.();
|
||
}, switchMsRef.current * jitter);
|
||
};
|
||
|
||
const scheduleNext = useCallback(() => scheduleRef.current?.(), []);
|
||
|
||
useEffect(() => {
|
||
if (autoRotate) scheduleNext();
|
||
else clearAuto();
|
||
return clearAuto;
|
||
}, [autoRotate, scheduleNext, clearAuto]);
|
||
|
||
const pick = useCallback((id) => {
|
||
clearAuto();
|
||
setExprId(id);
|
||
if (autoRotateRef.current) scheduleNext();
|
||
}, [clearAuto, scheduleNext]);
|
||
|
||
// ── Voice pipeline state ─────────────────────────────────────
|
||
const { isRecording, start: startRec, stop: stopRec } = usePushToTalk();
|
||
|
||
// voicePhase: 'idle' | 'listening' | 'thinking' | 'speaking'
|
||
const [voicePhase, setVoicePhase] = useState('idle');
|
||
const [voiceStatus, setVoiceStatus] = useState(null); // { type, text }
|
||
const [chatMessages, setChatMessages] = useState([]);
|
||
const [llmHistory, setLlmHistory] = useState([]); // last N {role,content} for context
|
||
const [memories, setMemories] = useState(() => getMemories());
|
||
const [isHermesMode, setIsHermesMode] = useState(false);
|
||
const voicePhaseRef = useRef('idle');
|
||
const isSpaceDownRef = useRef(false);
|
||
|
||
const refreshMemories = useCallback(() => setMemories(getMemories()), []);
|
||
|
||
// AbortController for the current voice pipeline (chat + TTS)
|
||
const abortRef = useRef(null);
|
||
|
||
const handleCancel = useCallback(() => {
|
||
isSpaceDownRef.current = false;
|
||
// Stop recorder if actively recording
|
||
const phase = voicePhaseRef.current;
|
||
if (phase === 'listening' || phase === 'acquiring') {
|
||
stopRec().catch(() => {});
|
||
}
|
||
abortRef.current?.abort();
|
||
abortRef.current = null;
|
||
setVoicePhase('idle');
|
||
setVoiceStatus(null);
|
||
setExprId('idle');
|
||
scheduleNext();
|
||
}, [scheduleNext, stopRec]);
|
||
|
||
// Load backend mode from server config on mount
|
||
useEffect(() => {
|
||
apiConfig().then(cfg => {
|
||
if (cfg?.backendMode === 'hermes') setIsHermesMode(true);
|
||
}).catch(() => {});
|
||
}, []);
|
||
|
||
useEffect(() => { voicePhaseRef.current = voicePhase; }, [voicePhase]);
|
||
|
||
const appendMsg = (role, text, audioBase64, audioMime, toolCalls) => {
|
||
const msg = { id: crypto.randomUUID(), role, text, audioBase64, audioMime, toolCalls };
|
||
setChatMessages(prev => [...prev.slice(-29), msg]); // keep last 30
|
||
return msg;
|
||
};
|
||
|
||
// ── Full push-to-talk flow ────────────────────────────────────
|
||
const handleSpaceDown = useCallback(async () => {
|
||
if (isSpaceDownRef.current || voicePhaseRef.current !== 'idle') return;
|
||
isSpaceDownRef.current = true;
|
||
|
||
clearAuto();
|
||
// Show "acquiring mic" state while getUserMedia is pending
|
||
setVoicePhase('acquiring');
|
||
setVoiceStatus({ type: 'acquiring', text: '⏳ 麦克风准备中…' });
|
||
|
||
try {
|
||
await startRec(); // resolves once stream is live and MediaRecorder.start() called
|
||
} catch (err) {
|
||
console.warn('[PTT] Mic acquisition failed:', err.message);
|
||
isSpaceDownRef.current = false;
|
||
setVoicePhase('idle');
|
||
setVoiceStatus({ type: 'error', text: '⚠ 麦克风不可用' });
|
||
setTimeout(() => setVoiceStatus(null), 2500);
|
||
scheduleNext();
|
||
return;
|
||
}
|
||
|
||
// Race condition guard: user released before mic was ready (fast tap)
|
||
if (!isSpaceDownRef.current) {
|
||
stopRec().catch(() => {}); // discard the just-acquired recording
|
||
setVoicePhase('idle');
|
||
setVoiceStatus(null);
|
||
scheduleNext();
|
||
return;
|
||
}
|
||
|
||
// Only show "listening" once the mic is actually recording
|
||
setExprId('listening');
|
||
setVoicePhase('listening');
|
||
setVoiceStatus({ type: 'listening', text: '🎙 请讲话 · 松开结束' });
|
||
}, [clearAuto, startRec, stopRec, scheduleNext]);
|
||
|
||
const handleSpaceUp = useCallback(async () => {
|
||
if (!isSpaceDownRef.current) return;
|
||
isSpaceDownRef.current = false;
|
||
// If still acquiring (Space released before mic was ready) — just cancel
|
||
if (voicePhaseRef.current === 'acquiring') {
|
||
setVoicePhase('idle');
|
||
setVoiceStatus(null);
|
||
scheduleNext();
|
||
return;
|
||
}
|
||
if (voicePhaseRef.current !== 'listening') return;
|
||
|
||
// ── Step 1: stop recording ────────────────────────────────
|
||
setVoicePhase('thinking');
|
||
setExprId('thinking');
|
||
setVoiceStatus({ type: 'thinking', text: '⚙ 识别中…' });
|
||
|
||
const { transcript: browserTranscript, audioBase64: userAudio, mimeType: userMime } = await stopRec();
|
||
|
||
// ── Transcribe with Volcengine ASR 2.0; fall back to browser SpeechRecognition ──
|
||
let userText = '';
|
||
if (userAudio) {
|
||
try {
|
||
setVoiceStatus({ type: 'thinking', text: '⚙ 识别中…' });
|
||
const { text } = await apiAsr(userAudio, userMime || 'audio/webm');
|
||
userText = text?.trim() || '';
|
||
} catch (err) {
|
||
console.warn('[ASR] Volcengine failed, using browser transcript:', err.message);
|
||
}
|
||
}
|
||
if (!userText) userText = browserTranscript.trim();
|
||
if (!userText) {
|
||
// Nothing recognised at all — silently abort
|
||
setVoicePhase('idle');
|
||
setExprId('idle');
|
||
setVoiceStatus(null);
|
||
return;
|
||
}
|
||
appendMsg('user', userText, userAudio, userMime);
|
||
setVoiceStatus({ type: 'thinking', text: '⚙ 思考中…' });
|
||
|
||
// Create a fresh AbortController for this pipeline run
|
||
const ac = new AbortController();
|
||
abortRef.current = ac;
|
||
|
||
// ── Step 2: LLM ─────────────────────────────────────────────
|
||
let botText = '';
|
||
let toolCalls = [];
|
||
try {
|
||
// In Hermes mode don't pass local memories — Hermes owns its own memory
|
||
const currentMemories = isHermesMode ? [] : getMemories().map(m => m.text);
|
||
const currentSettings = getSettings();
|
||
const newHistory = [...llmHistory, { role: 'user', content: userText }];
|
||
const result = await apiChat(newHistory.slice(-10), currentMemories, currentSettings, ac.signal);
|
||
botText = result.text;
|
||
toolCalls = result.toolCalls || [];
|
||
setLlmHistory([...newHistory, { role: 'assistant', content: botText }].slice(-20));
|
||
} catch (err) {
|
||
if (err.name === 'AbortError') {
|
||
// User cancelled — clean up and return without appending a bot message
|
||
setVoicePhase('idle');
|
||
setExprId('idle');
|
||
setVoiceStatus(null);
|
||
scheduleNext();
|
||
return;
|
||
}
|
||
// Friendly label for Hermes connection failures
|
||
const msg = err.message.includes('Cannot connect to Hermes')
|
||
? `⚠ Hermes Agent 无法连接，请检查 ${err.message.match(/at (.+?) —/)?.[1] ?? '地址'} 是否在线`
|
||
: `对话出错: ${err.message}`;
|
||
botText = msg;
|
||
}
|
||
|
||
// If aborted between LLM and TTS, bail out
|
||
if (ac.signal.aborted) {
|
||
setVoicePhase('idle'); setExprId('idle'); setVoiceStatus(null); scheduleNext(); return;
|
||
}
|
||
|
||
// ── Step 3: TTS ──────────────────────────────────────────────
|
||
setExprId('thinking');
|
||
setVoicePhase('speaking');
|
||
setVoiceStatus({ type: 'tts', text: '🔊 生成语音中…' });
|
||
|
||
const voice = getEffectiveVoice();
|
||
const speechText = stripMarkdownForSpeech(botText);
|
||
let botAudio = null;
|
||
let usingBrowserTts = false;
|
||
try {
|
||
const { audioBase64 } = await apiTts(speechText, voice);
|
||
botAudio = audioBase64;
|
||
} catch (err) {
|
||
console.warn('[TTS] Volcengine unavailable, using browser TTS:', err.message);
|
||
usingBrowserTts = true;
|
||
}
|
||
|
||
setVoiceStatus({ type: 'speaking', text: usingBrowserTts ? '🔊 播放中… (浏览器语音)' : '🔊 播放中…' });
|
||
|
||
appendMsg('bot', botText, botAudio, 'audio/mp3', toolCalls);
|
||
|
||
// Save to history
|
||
addEntry({
|
||
userText,
|
||
botText,
|
||
userAudioBase64: userAudio,
|
||
userAudioMime: userMime,
|
||
botAudioBase64: botAudio,
|
||
botAudioMime: 'audio/mp3',
|
||
});
|
||
|
||
// Background memory extraction — skip in Hermes mode (Hermes owns its memory)
|
||
if (!isHermesMode) {
|
||
apiExtractMemory(userText, botText).then(facts => {
|
||
let added = false;
|
||
facts.forEach(f => { if (addMemory(f)) added = true; });
|
||
if (added) refreshMemories();
|
||
}).catch(() => {});
|
||
}
|
||
|
||
// Play audio (only if not cancelled)
|
||
if (!ac.signal.aborted) {
|
||
setExprId('speaking');
|
||
if (botAudio) {
|
||
setVoiceStatus({ type: 'speaking', text: '🔊 播放中…' });
|
||
await playAudioBase64(botAudio, 'audio/mp3');
|
||
} else {
|
||
setVoiceStatus({ type: 'speaking', text: '🔊 播放中… (浏览器语音)' });
|
||
await speakText(speechText);
|
||
}
|
||
}
|
||
|
||
abortRef.current = null;
|
||
|
||
// ── Done: resume auto-rotate ─────────────────────────────────
|
||
setVoicePhase('idle');
|
||
setVoiceStatus(null);
|
||
const nextId = ALL_AUTO_IDS[Math.floor(Math.random() * ALL_AUTO_IDS.length)];
|
||
setExprId(nextId);
|
||
scheduleNext();
|
||
}, [stopRec, llmHistory, scheduleNext, isHermesMode, refreshMemories]);
|
||
|
||
// ── Keyboard handler ──────────────────────────────────────────
|
||
useEffect(() => {
|
||
function onKeyDown(e) {
|
||
if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA') return;
|
||
if (e.code === 'Space' && !e.repeat) {
|
||
e.preventDefault();
|
||
handleSpaceDown();
|
||
return;
|
||
}
|
||
if (voicePhaseRef.current === 'idle') {
|
||
const id = KEY_MAP[e.key.toLowerCase()];
|
||
if (id) pick(id);
|
||
}
|
||
}
|
||
function onKeyUp(e) {
|
||
if (e.code === 'Space') {
|
||
e.preventDefault();
|
||
handleSpaceUp();
|
||
}
|
||
}
|
||
window.addEventListener('keydown', onKeyDown);
|
||
window.addEventListener('keyup', onKeyUp);
|
||
return () => {
|
||
window.removeEventListener('keydown', onKeyDown);
|
||
window.removeEventListener('keyup', onKeyUp);
|
||
};
|
||
}, [handleSpaceDown, handleSpaceUp, pick]);
|
||
|
||
// ── Global mouse/touch fallback — catches release outside button boundary ──
|
||
useEffect(() => {
|
||
const onUp = () => handleSpaceUp();
|
||
window.addEventListener('mouseup', onUp);
|
||
window.addEventListener('touchend', onUp, { passive: true });
|
||
window.addEventListener('touchcancel', onUp, { passive: true });
|
||
return () => {
|
||
window.removeEventListener('mouseup', onUp);
|
||
window.removeEventListener('touchend', onUp);
|
||
window.removeEventListener('touchcancel', onUp);
|
||
};
|
||
}, [handleSpaceUp]);
|
||
|
||
// ── Safety timeout: auto-cancel if stuck in listening > 60 s ──
|
||
const recTimerRef = useRef(null);
|
||
useEffect(() => {
|
||
if (voicePhase === 'listening') {
|
||
recTimerRef.current = setTimeout(() => {
|
||
console.warn('[PTT] Auto-stop: max recording time reached');
|
||
handleSpaceUp();
|
||
}, 60_000);
|
||
} else {
|
||
clearTimeout(recTimerRef.current);
|
||
}
|
||
return () => clearTimeout(recTimerRef.current);
|
||
}, [voicePhase, handleSpaceUp]);
|
||
|
||
// ── Render ─────────────────────────────────────────────────
|
||
|
||
return (
|
||
<div className="app">
|
||
<header className="hdr">
|
||
<a href="/" className="hdr-home" title="返回主页">🏠</a>
|
||
<h1>J<span className="accent">BOT</span></h1>
|
||
<p className="subtitle">AI Robot Face · 240×240</p>
|
||
<nav className="hdr-nav">
|
||
<button className="hdr-link" onClick={() => navigate('/history')}>历史</button>
|
||
<button className="hdr-link" onClick={() => navigate('/settings')}>设置</button>
|
||
</nav>
|
||
</header>
|
||
|
||
<div className="page-body">
|
||
{/* ── Left column: face + controls ── */}
|
||
<aside className="left-col">
|
||
<section className="screen-wrap">
|
||
<div className="scale-controls">
|
||
<button className={`scale-btn${scale === 1 ? ' active' : ''}`} onClick={() => setScale(1)}>1×</button>
|
||
<button className={`scale-btn${scale === 2 ? ' active' : ''}`} onClick={() => setScale(2)}>2×</button>
|
||
</div>
|
||
<div className="bezel">
|
||
<RobotFace expressionId={exprId} scale={scale} />
|
||
</div>
|
||
<div className="expr-label">{byId[exprId]?.label}</div>
|
||
</section>
|
||
|
||
<section className="panel">
|
||
<div className="panel-section">
|
||
<h3>交互状态 <span className="key-hint">1 – 8</span></h3>
|
||
<div className="btn-grid">
|
||
{INTERACTION.map(e => (
|
||
<button
|
||
key={e.id}
|
||
className={`ebtn${exprId === e.id ? ' active' : ''}`}
|
||
style={{ '--c': e.color }}
|
||
onClick={() => pick(e.id)}
|
||
>{e.label}</button>
|
||
))}
|
||
</div>
|
||
</div>
|
||
|
||
<div className="panel-section">
|
||
<h3>微表情 <span className="key-hint">Q–I · 自动</span></h3>
|
||
<div className="btn-grid btn-grid-4">
|
||
{AUTO.map(e => (
|
||
<button
|
||
key={e.id}
|
||
className={`ebtn${exprId === e.id ? ' active' : ''}`}
|
||
style={{ '--c': e.color }}
|
||
onClick={() => pick(e.id)}
|
||
>{e.label}</button>
|
||
))}
|
||
</div>
|
||
</div>
|
||
|
||
<label className="toggle">
|
||
<input type="checkbox" checked={autoRotate} onChange={ev => setAutoRotate(ev.target.checked)} />
|
||
<span>自动轮换微表情</span>
|
||
</label>
|
||
</section>
|
||
</aside>
|
||
|
||
{/* ── Right column: memory + chat + bottom bar ── */}
|
||
<div className="right-col">
|
||
{!isHermesMode && (
|
||
<MemoryPanel memories={memories} onMemoriesChange={refreshMemories} />
|
||
)}
|
||
{isHermesMode && (
|
||
<div className="hermes-mode-badge">🧠 Hermes Agent 模式 · 记忆由 Agent 管理</div>
|
||
)}
|
||
<ChatPanel messages={chatMessages} voiceStatus={voiceStatus} />
|
||
<VoiceButton phase={voicePhase} onStart={handleSpaceDown} onEnd={handleSpaceUp}
|
||
onCancel={handleCancel} />
|
||
</div>
|
||
</div>
|
||
</div>
|
||
);
|
||
}
|