import { useState, useEffect, useRef, useCallback } from 'react'; import { useNavigate } from 'react-router-dom'; import RobotFace from './components/RobotFace'; import ChatPanel from './components/ChatPanel'; import MemoryPanel from './components/MemoryPanel'; import { EXPRESSIONS, ALL_AUTO_IDS, byId } from './data/expressions'; import { apiChat, apiTts, apiAsr, apiExtractMemory, apiConfig } from './services/api'; import { usePushToTalk } from './hooks/usePushToTalk'; import { addEntry } from './store/historyStore'; import { getMemories, addMemory } from './store/memoryStore'; import { getSettings, getEffectiveVoice } from './store/settingsStore'; import { VOICES, getVoiceLabel } from './data/voices'; import './App.css'; const INTERACTION = EXPRESSIONS.filter(e => e.category === 'interaction'); const AUTO = EXPRESSIONS.filter(e => e.category === 'auto'); const KEY_MAP = { '1': 'idle', '2': 'listening', '3': 'thinking', '4': 'speaking', '5': 'happy', '6': 'error', '7': 'confused', '8': 'dreaming', 'q': 'natural_smile', 'w': 'calm_relax', 'e': 'curious_observe', 'r': 'light_joy', 't': 'side_think', 'y': 'warm_friendly', 'u': 'quiet_listen', 'i': 'focus_gaze', }; // Play base64-encoded audio blob; resolves when playback ends or on error async function playAudioBase64(base64, mimeType = 'audio/mp3') { return new Promise((resolve) => { const audio = new Audio(`data:${mimeType};base64,${base64}`); audio.onended = resolve; audio.onerror = resolve; // don't block on error audio.play().catch(resolve); }); } // Browser TTS fallback function speakText(text) { return new Promise((resolve) => { if (!window.speechSynthesis) return resolve(); const utt = new SpeechSynthesisUtterance(text); utt.lang = 'zh-CN'; utt.rate = 1.05; utt.onend = resolve; utt.onerror = resolve; window.speechSynthesis.speak(utt); }); } // โ”€โ”€ Hold-to-talk voice button (defined outside App to keep stable identity) โ”€โ”€ function VoiceButton({ phase, onStart, onEnd }) { const busy = phase === 'thinking' || phase === 'speaking'; let label = '๐ŸŽ™ ๆŒ‰ไฝ่ฏด่ฏ'; let mod = ''; if (phase === 'acquiring') { label = 'โณ ๅ‡†ๅค‡ไธญโ€ฆ'; mod = ' vbtn--acquiring'; } else if (phase === 'listening') { label = '๐Ÿ”ด ๆพๅผ€ๅ‘้€'; mod = ' vbtn--recording'; } else if (phase === 'thinking') { label = 'โš™ ๆ€่€ƒไธญโ€ฆ'; mod = ' vbtn--busy'; } else if (phase === 'speaking') { label = '๐Ÿ”Š ๆ’ญๆ”พไธญโ€ฆ'; mod = ' vbtn--busy'; } const startEvt = (e) => { e.preventDefault(); if (!busy) onStart(); }; const endEvt = (e) => { e.preventDefault(); onEnd(); }; return (
); } export default function App() { const navigate = useNavigate(); // โ”€โ”€ Expression / auto-rotate state โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ const [exprId, setExprId] = useState(ALL_AUTO_IDS[0]); const [autoRotate, setAutoRotate] = useState(true); const [scale, setScale] = useState(1); // switchMs is read from settings on mount (changed only via Settings page) const [switchMs] = useState(() => getSettings().switchIntervalMs ?? 10000); const autoTimerRef = useRef(null); const autoRotateRef = useRef(true); const exprIdRef = useRef(ALL_AUTO_IDS[0]); const scheduleRef = useRef(null); const switchMsRef = useRef(switchMs); useEffect(() => { autoRotateRef.current = autoRotate; }, [autoRotate]); useEffect(() => { exprIdRef.current = exprId; }, [exprId]); const clearAuto = useCallback(() => { clearTimeout(autoTimerRef.current); autoTimerRef.current = null; }, []); scheduleRef.current = () => { clearAuto(); if (!autoRotateRef.current) return; const jitter = 1 + (Math.random() - 0.5) * 0.3; autoTimerRef.current = setTimeout(() => { const candidates = ALL_AUTO_IDS.filter(id => id !== exprIdRef.current); const nextId = candidates[Math.floor(Math.random() * candidates.length)]; setExprId(nextId); scheduleRef.current?.(); }, switchMsRef.current * jitter); }; const scheduleNext = useCallback(() => scheduleRef.current?.(), []); useEffect(() => { if (autoRotate) scheduleNext(); else clearAuto(); return clearAuto; }, [autoRotate, scheduleNext, clearAuto]); const pick = useCallback((id) => { clearAuto(); setExprId(id); if (autoRotateRef.current) scheduleNext(); }, [clearAuto, scheduleNext]); // โ”€โ”€ Voice pipeline state โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ const { isRecording, start: startRec, stop: stopRec } = usePushToTalk(); // voicePhase: 'idle' | 'listening' | 'thinking' | 'speaking' const [voicePhase, setVoicePhase] = useState('idle'); const [voiceStatus, setVoiceStatus] = useState(null); // { type, text } const [chatMessages, setChatMessages] = useState([]); const [llmHistory, setLlmHistory] = useState([]); // last N {role,content} for context const [memories, setMemories] = useState(() => getMemories()); const [isHermesMode, setIsHermesMode] = useState(false); const voicePhaseRef = useRef('idle'); const isSpaceDownRef = useRef(false); const refreshMemories = useCallback(() => setMemories(getMemories()), []); // Load backend mode from server config on mount useEffect(() => { apiConfig().then(cfg => { if (cfg?.backendMode === 'hermes') setIsHermesMode(true); }).catch(() => {}); }, []); useEffect(() => { voicePhaseRef.current = voicePhase; }, [voicePhase]); const appendMsg = (role, text, audioBase64, audioMime, toolCalls) => { const msg = { id: crypto.randomUUID(), role, text, audioBase64, audioMime, toolCalls }; setChatMessages(prev => [...prev.slice(-29), msg]); // keep last 30 return msg; }; // โ”€โ”€ Full push-to-talk flow โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ const handleSpaceDown = useCallback(async () => { if (isSpaceDownRef.current || voicePhaseRef.current !== 'idle') return; isSpaceDownRef.current = true; clearAuto(); // Show "acquiring mic" state while getUserMedia is pending setVoicePhase('acquiring'); setVoiceStatus({ type: 'acquiring', text: 'โณ ้บฆๅ…‹้ฃŽๅ‡†ๅค‡ไธญโ€ฆ' }); try { await startRec(); // resolves once stream is live and MediaRecorder.start() called } catch (err) { console.warn('[PTT] Mic acquisition failed:', err.message); isSpaceDownRef.current = false; setVoicePhase('idle'); setVoiceStatus({ type: 'error', text: 'โš  ้บฆๅ…‹้ฃŽไธๅฏ็”จ' }); setTimeout(() => setVoiceStatus(null), 2500); scheduleNext(); return; } // Only show "listening" once the mic is actually recording setExprId('listening'); setVoicePhase('listening'); setVoiceStatus({ type: 'listening', text: '๐ŸŽ™ ่ฏท่ฎฒ่ฏ ยท ๆพๅผ€็ป“ๆŸ' }); }, [clearAuto, startRec, scheduleNext]); const handleSpaceUp = useCallback(async () => { if (!isSpaceDownRef.current) return; isSpaceDownRef.current = false; // If still acquiring (Space released before mic was ready) โ€” just cancel if (voicePhaseRef.current === 'acquiring') { setVoicePhase('idle'); setVoiceStatus(null); scheduleNext(); return; } if (voicePhaseRef.current !== 'listening') return; // โ”€โ”€ Step 1: stop recording โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ setVoicePhase('thinking'); setExprId('thinking'); setVoiceStatus({ type: 'thinking', text: 'โš™ ่ฏ†ๅˆซไธญโ€ฆ' }); const { transcript: browserTranscript, audioBase64: userAudio, mimeType: userMime } = await stopRec(); // โ”€โ”€ Transcribe with Volcengine ASR 2.0; fall back to browser SpeechRecognition โ”€โ”€ let userText = ''; if (userAudio) { try { setVoiceStatus({ type: 'thinking', text: 'โš™ ่ฏ†ๅˆซไธญโ€ฆ' }); const { text } = await apiAsr(userAudio, userMime || 'audio/webm'); userText = text?.trim() || ''; } catch (err) { console.warn('[ASR] Volcengine failed, using browser transcript:', err.message); } } if (!userText) userText = browserTranscript.trim(); if (!userText) { // Nothing recognised at all โ€” silently abort setVoicePhase('idle'); setExprId('idle'); setVoiceStatus(null); return; } appendMsg('user', userText, userAudio, userMime); setVoiceStatus({ type: 'thinking', text: 'โš™ ๆ€่€ƒไธญโ€ฆ' }); // โ”€โ”€ Step 2: LLM โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ let botText = ''; let toolCalls = []; try { // In Hermes mode don't pass local memories โ€” Hermes owns its own memory const currentMemories = isHermesMode ? [] : getMemories().map(m => m.text); const currentSettings = getSettings(); const newHistory = [...llmHistory, { role: 'user', content: userText }]; const result = await apiChat(newHistory.slice(-10), currentMemories, currentSettings); botText = result.text; toolCalls = result.toolCalls || []; setLlmHistory([...newHistory, { role: 'assistant', content: botText }].slice(-20)); } catch (err) { botText = `ๅฏน่ฏๅ‡บ้”™: ${err.message}`; } // โ”€โ”€ Step 3: TTS โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ setExprId('speaking'); setVoicePhase('speaking'); setVoiceStatus({ type: 'speaking', text: '๐Ÿ”Š ๆ’ญๆ”พไธญโ€ฆ' }); const voice = getEffectiveVoice(); let botAudio = null; let usingBrowserTts = false; try { const { audioBase64 } = await apiTts(botText, voice); botAudio = audioBase64; } catch (err) { console.warn('[TTS] Volcengine unavailable, using browser TTS:', err.message); usingBrowserTts = true; } if (usingBrowserTts) { setVoiceStatus({ type: 'speaking', text: '๐Ÿ”Š ๆ’ญๆ”พไธญโ€ฆ (ๆต่งˆๅ™จ่ฏญ้Ÿณ)' }); } appendMsg('bot', botText, botAudio, 'audio/mp3', toolCalls); // Save to history addEntry({ userText, botText, userAudioBase64: userAudio, userAudioMime: userMime, botAudioBase64: botAudio, botAudioMime: 'audio/mp3', }); // Background memory extraction โ€” skip in Hermes mode (Hermes owns its memory) if (!isHermesMode) { apiExtractMemory(userText, botText).then(facts => { let added = false; facts.forEach(f => { if (addMemory(f)) added = true; }); if (added) refreshMemories(); }).catch(() => {}); } // Play audio if (botAudio) { await playAudioBase64(botAudio, 'audio/mp3'); } else { await speakText(botText); } // โ”€โ”€ Done: resume auto-rotate โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ setVoicePhase('idle'); setVoiceStatus(null); const nextId = ALL_AUTO_IDS[Math.floor(Math.random() * ALL_AUTO_IDS.length)]; setExprId(nextId); scheduleNext(); }, [stopRec, llmHistory, scheduleNext]); // โ”€โ”€ Keyboard handler โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ useEffect(() => { function onKeyDown(e) { if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA') return; if (e.code === 'Space' && !e.repeat) { e.preventDefault(); handleSpaceDown(); return; } if (voicePhaseRef.current === 'idle') { const id = KEY_MAP[e.key.toLowerCase()]; if (id) pick(id); } } function onKeyUp(e) { if (e.code === 'Space') { e.preventDefault(); handleSpaceUp(); } } window.addEventListener('keydown', onKeyDown); window.addEventListener('keyup', onKeyUp); return () => { window.removeEventListener('keydown', onKeyDown); window.removeEventListener('keyup', onKeyUp); }; }, [handleSpaceDown, handleSpaceUp, pick]); // โ”€โ”€ Render โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ return (
๐Ÿ 

JBOT

AI Robot Face ยท 240ร—240

{/* โ”€โ”€ Left column: face + controls โ”€โ”€ */} {/* โ”€โ”€ Right column: memory + chat + bottom bar โ”€โ”€ */}
{!isHermesMode && ( )} {isHermesMode && (
๐Ÿง  Hermes Agent ๆจกๅผ ยท ่ฎฐๅฟ†็”ฑ Agent ็ฎก็†
)}
); }