import { useState, useEffect, useRef, useCallback } from 'react';
import { useNavigate } from 'react-router-dom';
import RobotFace from './components/RobotFace';
import ChatPanel from './components/ChatPanel';
import MemoryPanel from './components/MemoryPanel';
import { EXPRESSIONS, ALL_AUTO_IDS, byId } from './data/expressions';
import { apiChat, apiTts, apiAsr, apiExtractMemory, apiConfig } from './services/api';
import { usePushToTalk } from './hooks/usePushToTalk';
import { addEntry } from './store/historyStore';
import { getMemories, addMemory } from './store/memoryStore';
import { getSettings, getEffectiveVoice } from './store/settingsStore';
import { VOICES, getVoiceLabel } from './data/voices';
import './App.css';
const INTERACTION = EXPRESSIONS.filter(e => e.category === 'interaction');
const AUTO = EXPRESSIONS.filter(e => e.category === 'auto');
const KEY_MAP = {
'1': 'idle', '2': 'listening', '3': 'thinking', '4': 'speaking',
'5': 'happy', '6': 'error', '7': 'confused', '8': 'dreaming',
'q': 'natural_smile', 'w': 'calm_relax', 'e': 'curious_observe', 'r': 'light_joy',
't': 'side_think', 'y': 'warm_friendly', 'u': 'quiet_listen', 'i': 'focus_gaze',
};
// Play base64-encoded audio blob; resolves when playback ends or on error
async function playAudioBase64(base64, mimeType = 'audio/mp3') {
return new Promise((resolve) => {
const audio = new Audio(`data:${mimeType};base64,${base64}`);
audio.onended = resolve;
audio.onerror = resolve; // don't block on error
audio.play().catch(resolve);
});
}
// Browser TTS fallback
function speakText(text) {
return new Promise((resolve) => {
if (!window.speechSynthesis) return resolve();
const utt = new SpeechSynthesisUtterance(text);
utt.lang = 'zh-CN';
utt.rate = 1.05;
utt.onend = resolve;
utt.onerror = resolve;
window.speechSynthesis.speak(utt);
});
}
// โโ Hold-to-talk voice button (defined outside App to keep stable identity) โโ
function VoiceButton({ phase, onStart, onEnd }) {
const busy = phase === 'thinking' || phase === 'speaking';
let label = '๐ ๆไฝ่ฏด่ฏ';
let mod = '';
if (phase === 'acquiring') { label = 'โณ ๅๅคไธญโฆ'; mod = ' vbtn--acquiring'; }
else if (phase === 'listening') { label = '๐ด ๆพๅผๅ้'; mod = ' vbtn--recording'; }
else if (phase === 'thinking') { label = 'โ ๆ่ไธญโฆ'; mod = ' vbtn--busy'; }
else if (phase === 'speaking') { label = '๐ ๆญๆพไธญโฆ'; mod = ' vbtn--busy'; }
const startEvt = (e) => { e.preventDefault(); if (!busy) onStart(); };
const endEvt = (e) => { e.preventDefault(); onEnd(); };
return (
);
}
export default function App() {
const navigate = useNavigate();
// โโ Expression / auto-rotate state โโโโโโโโโโโโโโโโโโโโโโโโโโโ
const [exprId, setExprId] = useState(ALL_AUTO_IDS[0]);
const [autoRotate, setAutoRotate] = useState(true);
const [scale, setScale] = useState(1);
// switchMs is read from settings on mount (changed only via Settings page)
const [switchMs] = useState(() => getSettings().switchIntervalMs ?? 10000);
const autoTimerRef = useRef(null);
const autoRotateRef = useRef(true);
const exprIdRef = useRef(ALL_AUTO_IDS[0]);
const scheduleRef = useRef(null);
const switchMsRef = useRef(switchMs);
useEffect(() => { autoRotateRef.current = autoRotate; }, [autoRotate]);
useEffect(() => { exprIdRef.current = exprId; }, [exprId]);
const clearAuto = useCallback(() => {
clearTimeout(autoTimerRef.current);
autoTimerRef.current = null;
}, []);
scheduleRef.current = () => {
clearAuto();
if (!autoRotateRef.current) return;
const jitter = 1 + (Math.random() - 0.5) * 0.3;
autoTimerRef.current = setTimeout(() => {
const candidates = ALL_AUTO_IDS.filter(id => id !== exprIdRef.current);
const nextId = candidates[Math.floor(Math.random() * candidates.length)];
setExprId(nextId);
scheduleRef.current?.();
}, switchMsRef.current * jitter);
};
const scheduleNext = useCallback(() => scheduleRef.current?.(), []);
useEffect(() => {
if (autoRotate) scheduleNext();
else clearAuto();
return clearAuto;
}, [autoRotate, scheduleNext, clearAuto]);
const pick = useCallback((id) => {
clearAuto();
setExprId(id);
if (autoRotateRef.current) scheduleNext();
}, [clearAuto, scheduleNext]);
// โโ Voice pipeline state โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
const { isRecording, start: startRec, stop: stopRec } = usePushToTalk();
// voicePhase: 'idle' | 'listening' | 'thinking' | 'speaking'
const [voicePhase, setVoicePhase] = useState('idle');
const [voiceStatus, setVoiceStatus] = useState(null); // { type, text }
const [chatMessages, setChatMessages] = useState([]);
const [llmHistory, setLlmHistory] = useState([]); // last N {role,content} for context
const [memories, setMemories] = useState(() => getMemories());
const [isHermesMode, setIsHermesMode] = useState(false);
const voicePhaseRef = useRef('idle');
const isSpaceDownRef = useRef(false);
const refreshMemories = useCallback(() => setMemories(getMemories()), []);
// Load backend mode from server config on mount
useEffect(() => {
apiConfig().then(cfg => {
if (cfg?.backendMode === 'hermes') setIsHermesMode(true);
}).catch(() => {});
}, []);
useEffect(() => { voicePhaseRef.current = voicePhase; }, [voicePhase]);
const appendMsg = (role, text, audioBase64, audioMime, toolCalls) => {
const msg = { id: crypto.randomUUID(), role, text, audioBase64, audioMime, toolCalls };
setChatMessages(prev => [...prev.slice(-29), msg]); // keep last 30
return msg;
};
// โโ Full push-to-talk flow โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
const handleSpaceDown = useCallback(async () => {
if (isSpaceDownRef.current || voicePhaseRef.current !== 'idle') return;
isSpaceDownRef.current = true;
clearAuto();
// Show "acquiring mic" state while getUserMedia is pending
setVoicePhase('acquiring');
setVoiceStatus({ type: 'acquiring', text: 'โณ ้บฆๅ
้ฃๅๅคไธญโฆ' });
try {
await startRec(); // resolves once stream is live and MediaRecorder.start() called
} catch (err) {
console.warn('[PTT] Mic acquisition failed:', err.message);
isSpaceDownRef.current = false;
setVoicePhase('idle');
setVoiceStatus({ type: 'error', text: 'โ ้บฆๅ
้ฃไธๅฏ็จ' });
setTimeout(() => setVoiceStatus(null), 2500);
scheduleNext();
return;
}
// Only show "listening" once the mic is actually recording
setExprId('listening');
setVoicePhase('listening');
setVoiceStatus({ type: 'listening', text: '๐ ่ฏท่ฎฒ่ฏ ยท ๆพๅผ็ปๆ' });
}, [clearAuto, startRec, scheduleNext]);
const handleSpaceUp = useCallback(async () => {
if (!isSpaceDownRef.current) return;
isSpaceDownRef.current = false;
// If still acquiring (Space released before mic was ready) โ just cancel
if (voicePhaseRef.current === 'acquiring') {
setVoicePhase('idle');
setVoiceStatus(null);
scheduleNext();
return;
}
if (voicePhaseRef.current !== 'listening') return;
// โโ Step 1: stop recording โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
setVoicePhase('thinking');
setExprId('thinking');
setVoiceStatus({ type: 'thinking', text: 'โ ่ฏๅซไธญโฆ' });
const { transcript: browserTranscript, audioBase64: userAudio, mimeType: userMime } = await stopRec();
// โโ Transcribe with Volcengine ASR 2.0; fall back to browser SpeechRecognition โโ
let userText = '';
if (userAudio) {
try {
setVoiceStatus({ type: 'thinking', text: 'โ ่ฏๅซไธญโฆ' });
const { text } = await apiAsr(userAudio, userMime || 'audio/webm');
userText = text?.trim() || '';
} catch (err) {
console.warn('[ASR] Volcengine failed, using browser transcript:', err.message);
}
}
if (!userText) userText = browserTranscript.trim();
if (!userText) {
// Nothing recognised at all โ silently abort
setVoicePhase('idle');
setExprId('idle');
setVoiceStatus(null);
return;
}
appendMsg('user', userText, userAudio, userMime);
setVoiceStatus({ type: 'thinking', text: 'โ ๆ่ไธญโฆ' });
// โโ Step 2: LLM โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
let botText = '';
let toolCalls = [];
try {
// In Hermes mode don't pass local memories โ Hermes owns its own memory
const currentMemories = isHermesMode ? [] : getMemories().map(m => m.text);
const currentSettings = getSettings();
const newHistory = [...llmHistory, { role: 'user', content: userText }];
const result = await apiChat(newHistory.slice(-10), currentMemories, currentSettings);
botText = result.text;
toolCalls = result.toolCalls || [];
setLlmHistory([...newHistory, { role: 'assistant', content: botText }].slice(-20));
} catch (err) {
botText = `ๅฏน่ฏๅบ้: ${err.message}`;
}
// โโ Step 3: TTS โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
setExprId('speaking');
setVoicePhase('speaking');
setVoiceStatus({ type: 'speaking', text: '๐ ๆญๆพไธญโฆ' });
const voice = getEffectiveVoice();
let botAudio = null;
let usingBrowserTts = false;
try {
const { audioBase64 } = await apiTts(botText, voice);
botAudio = audioBase64;
} catch (err) {
console.warn('[TTS] Volcengine unavailable, using browser TTS:', err.message);
usingBrowserTts = true;
}
if (usingBrowserTts) {
setVoiceStatus({ type: 'speaking', text: '๐ ๆญๆพไธญโฆ (ๆต่งๅจ่ฏญ้ณ)' });
}
appendMsg('bot', botText, botAudio, 'audio/mp3', toolCalls);
// Save to history
addEntry({
userText,
botText,
userAudioBase64: userAudio,
userAudioMime: userMime,
botAudioBase64: botAudio,
botAudioMime: 'audio/mp3',
});
// Background memory extraction โ skip in Hermes mode (Hermes owns its memory)
if (!isHermesMode) {
apiExtractMemory(userText, botText).then(facts => {
let added = false;
facts.forEach(f => { if (addMemory(f)) added = true; });
if (added) refreshMemories();
}).catch(() => {});
}
// Play audio
if (botAudio) {
await playAudioBase64(botAudio, 'audio/mp3');
} else {
await speakText(botText);
}
// โโ Done: resume auto-rotate โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
setVoicePhase('idle');
setVoiceStatus(null);
const nextId = ALL_AUTO_IDS[Math.floor(Math.random() * ALL_AUTO_IDS.length)];
setExprId(nextId);
scheduleNext();
}, [stopRec, llmHistory, scheduleNext]);
// โโ Keyboard handler โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
useEffect(() => {
function onKeyDown(e) {
if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA') return;
if (e.code === 'Space' && !e.repeat) {
e.preventDefault();
handleSpaceDown();
return;
}
if (voicePhaseRef.current === 'idle') {
const id = KEY_MAP[e.key.toLowerCase()];
if (id) pick(id);
}
}
function onKeyUp(e) {
if (e.code === 'Space') {
e.preventDefault();
handleSpaceUp();
}
}
window.addEventListener('keydown', onKeyDown);
window.addEventListener('keyup', onKeyUp);
return () => {
window.removeEventListener('keydown', onKeyDown);
window.removeEventListener('keyup', onKeyUp);
};
}, [handleSpaceDown, handleSpaceUp, pick]);
// โโ Render โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
return (
{/* โโ Left column: face + controls โโ */}
{/* โโ Right column: memory + chat + bottom bar โโ */}
{!isHermesMode && (
)}
{isHermesMode && (
๐ง Hermes Agent ๆจกๅผ ยท ่ฎฐๅฟ็ฑ Agent ็ฎก็
)}
);
}