Files
junvandCopilot bb3fe1b554 jbot: disable local memory in Hermes Agent mode
In Hermes mode JBOT is a pure voice UI — memory is owned by the Agent:
- Backend: sys prompt excludes local memories when backend_mode=hermes
- Backend: config_view now returns backendMode so the frontend can adapt
- Frontend: App.jsx loads backendMode on mount via apiConfig()
- Frontend: memories not passed to apiChat() when isHermesMode
- Frontend: memory extraction (apiExtractMemory) skipped when isHermesMode
- Frontend: MemoryPanel hidden in Hermes mode; replaced by subtle
  '🧠 Hermes Agent 模式 · 记忆由 Agent 管理' badge

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-05-17 14:13:37 +10:00

632 lines
23 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
JBOT backend views — Python port of the Node.js server/index.js
Endpoints:
GET /jbot/ → SPA shell (index.html)
POST /api/jbot/chat/ → OpenRouter LLM proxy
POST /api/jbot/tts/ → Volcengine seed-tts-2.0
POST /api/jbot/asr/ → Volcengine BigASR 1.0 (WebSocket binary protocol)
POST /api/jbot/memory/extract/ → background memory extraction via LLM
GET /api/jbot/config/ → public config flags
GET /api/jbot/api-config/ → read API credentials/settings (tokens masked)
POST /api/jbot/api-config/ → save API credentials/settings to DB
"""
import base64
import gzip
import json
import logging
import os
import struct
import subprocess
import threading
import uuid
import requests
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import TemplateView
logger = logging.getLogger(__name__)
# ── SPA shell ────────────────────────────────────────────────────────────────
class JbotIndexView(TemplateView):
template_name = 'jbot/index.html'
# ── Helper: require JSON POST ─────────────────────────────────────────────────
def _json_body(request):
try:
return json.loads(request.body)
except Exception:
return {}
# ── Config helper: DB values override env vars ────────────────────────────────
def _get_cfg():
"""Return merged config: DB row first, env vars as fallback."""
defaults = {
'volc_app_id': os.environ.get('VOLC_APP_ID', ''),
'volc_access_token': os.environ.get('VOLC_ACCESS_TOKEN', ''),
'openrouter_api_key': os.environ.get('OPENROUTER_API_KEY', ''),
'llm_model': os.environ.get('LLM_MODEL', 'deepseek/deepseek-chat'),
'volc_tts_voice': os.environ.get('VOLC_TTS_VOICE', 'zh_female_vv_uranus_bigtts'),
'volc_asr_resource': os.environ.get('VOLC_ASR_RESOURCE', 'volc.bigasr.sauc.duration'),
'hermes_url': os.environ.get('HERMES_URL', 'http://192.168.1.10:8642'),
'hermes_api_key': os.environ.get('HERMES_API_KEY', ''),
'backend_mode': os.environ.get('BACKEND_MODE', 'openrouter'),
}
try:
from .models import JbotApiConfig
db = JbotApiConfig.load()
for k in defaults:
db_val = getattr(db, k, '')
if db_val:
defaults[k] = db_val
except Exception:
pass
return defaults
# ── API Config CRUD (read/write credentials via UI) ──────────────────────────
@csrf_exempt
def api_config_view(request):
if request.method == 'GET':
try:
from .models import JbotApiConfig
db = JbotApiConfig.load()
env = {
'volc_app_id': os.environ.get('VOLC_APP_ID', ''),
'volc_access_token': os.environ.get('VOLC_ACCESS_TOKEN', ''),
'openrouter_api_key': os.environ.get('OPENROUTER_API_KEY', ''),
}
def _masked(db_val, env_val):
return '***' if (db_val or env_val) else ''
return JsonResponse({
'volcAppId': db.volc_app_id or env['volc_app_id'],
'volcAccessToken': _masked(db.volc_access_token, env['volc_access_token']),
'openrouterApiKey': _masked(db.openrouter_api_key, env['openrouter_api_key']),
'llmModel': db.llm_model or os.environ.get('LLM_MODEL', 'deepseek/deepseek-chat'),
'volcTtsVoice': db.volc_tts_voice or os.environ.get('VOLC_TTS_VOICE', 'zh_female_vv_uranus_bigtts'),
'volcAsrResource': db.volc_asr_resource or os.environ.get('VOLC_ASR_RESOURCE', 'volc.bigasr.sauc.duration'),
'hermesUrl': db.hermes_url or os.environ.get('HERMES_URL', 'http://192.168.1.10:8642'),
'hermesApiKey': _masked(db.hermes_api_key, os.environ.get('HERMES_API_KEY', '')),
'backendMode': db.backend_mode or os.environ.get('BACKEND_MODE', 'openrouter'),
'hasVolc': bool(db.volc_app_id or env['volc_app_id']),
'hasLlm': bool(db.openrouter_api_key or env['openrouter_api_key']),
})
except Exception as e:
return JsonResponse({'error': str(e)}, status=500)
if request.method == 'POST':
data = _json_body(request)
try:
from .models import JbotApiConfig
db = JbotApiConfig.load()
# Only update fields that are present and not the masked placeholder
mapping = {
'volcAppId': 'volc_app_id',
'llmModel': 'llm_model',
'volcTtsVoice': 'volc_tts_voice',
'volcAsrResource': 'volc_asr_resource',
'hermesUrl': 'hermes_url',
'backendMode': 'backend_mode',
}
for json_key, db_field in mapping.items():
if json_key in data:
setattr(db, db_field, data[json_key])
# Secret fields: only update if non-empty and not the placeholder
for json_key, db_field in [('volcAccessToken', 'volc_access_token'),
('openrouterApiKey', 'openrouter_api_key'),
('hermesApiKey', 'hermes_api_key')]:
val = data.get(json_key, '')
if val and val != '***':
setattr(db, db_field, val)
db.save()
return JsonResponse({'ok': True})
except Exception as e:
logger.error('[API-CONFIG] save failed: %s', e)
return JsonResponse({'error': str(e)}, status=500)
return JsonResponse({'error': 'GET or POST required'}, status=405)
# ── TTS 2.0 (Volcengine seed-tts-2.0 HTTP streaming NDJSON) ─────────────────
def _do_tts(text, voice=None):
cfg = _get_cfg()
app_id = cfg['volc_app_id']
token = cfg['volc_access_token']
speaker = voice or cfg['volc_tts_voice']
resource_id = os.environ.get('VOLC_TTS_RESOURCE', 'seed-tts-2.0')
if not app_id or not token:
raise ValueError('Volcengine credentials not configured — set them in JBOT Settings → API 配置')
resp = requests.post(
'https://openspeech.bytedance.com/api/v3/tts/unidirectional',
headers={
'Content-Type': 'application/json',
'X-Api-App-Id': app_id,
'X-Api-Access-Key': token,
'X-Api-Resource-Id': resource_id,
'X-Api-Request-Id': str(uuid.uuid4()),
},
json={
'user': {'uid': 'jbot-user'},
'req_params': {
'text': text,
'speaker': speaker,
'audio_params': {'format': 'mp3', 'sample_rate': 24000, 'speech_rate': 0},
},
},
stream=True,
timeout=30,
)
if not resp.ok:
raise RuntimeError(f'TTS HTTP {resp.status_code}: {resp.text[:200]}')
audio_chunks = []
for line in resp.iter_lines():
if not line:
continue
try:
obj = json.loads(line)
except Exception:
continue
if obj.get('code') == 0 and obj.get('data'):
audio_chunks.append(base64.b64decode(obj['data']))
elif obj.get('code') == 20000000:
break
elif obj.get('code', 0) != 0:
raise RuntimeError(f"TTS error code {obj.get('code')}: {obj.get('message', '')}")
if not audio_chunks:
raise RuntimeError('TTS returned no audio data')
return b''.join(audio_chunks)
@csrf_exempt
def tts_view(request):
if request.method != 'POST':
return JsonResponse({'error': 'POST required'}, status=405)
data = _json_body(request)
text = data.get('text', '').strip()
voice = data.get('voice', '')
if not text:
return JsonResponse({'error': 'text required'}, status=400)
try:
audio_bytes = _do_tts(text, voice or None)
return JsonResponse({
'audioBase64': base64.b64encode(audio_bytes).decode(),
'format': 'mp3',
})
except Exception as e:
logger.error('[TTS] %s', e)
return JsonResponse({'error': str(e)}, status=500)
# ── ASR 1.0 (Volcengine BigASR WebSocket binary protocol) ────────────────────
_ASR_MT_FULL_CLIENT = 1
_ASR_MT_AUDIO_ONLY = 2
_ASR_MT_FULL_SERVER = 9
_ASR_MT_ERROR = 15
_FLAG_NO_SEQ = 0
_FLAG_POS_SEQ = 1
_FLAG_LAST_NO = 2
_FLAG_NEG_SEQ = 3
_SER_NONE = 0
_SER_JSON = 1
_CMP_NONE = 0
_CMP_GZIP = 1
def _asr_pack(msg_type, flag, serialization=_SER_NONE, compression=_CMP_NONE,
payload=b'', sequence=None):
header = bytes([0x11, (msg_type << 4) | flag, (serialization << 4) | compression, 0x00])
parts = [header]
if flag in (_FLAG_POS_SEQ, _FLAG_NEG_SEQ):
parts.append(struct.pack('>i', sequence or 0))
parts.append(struct.pack('>I', len(payload)))
parts.append(payload)
return b''.join(parts)
def _asr_unpack(data: bytes):
header_bytes = (data[0] & 0x0F) * 4
msg_type = (data[1] >> 4) & 0x0F
flag = data[1] & 0x0F
compression = data[2] & 0x0F
off = header_bytes
if flag in (_FLAG_POS_SEQ, _FLAG_NEG_SEQ):
off += 4
if msg_type == _ASR_MT_ERROR:
code = struct.unpack_from('>I', data, off)[0]; off += 4
msg_len = struct.unpack_from('>I', data, off)[0]; off += 4
msg = data[off:off + msg_len].decode('utf-8', errors='replace')
raise RuntimeError(f'ASR error {code}: {msg}')
payload = b''
if off + 4 <= len(data):
p_len = struct.unpack_from('>I', data, off)[0]; off += 4
if p_len > 0:
payload = data[off:off + p_len]
if compression == _CMP_GZIP and payload:
payload = gzip.decompress(payload)
try:
return json.loads(payload.decode('utf-8'))
except Exception:
return None
def _to_pcm(audio_bytes: bytes, mime_type: str = 'audio/webm') -> bytes:
fmt = mime_type.split('/')[1].split(';')[0].strip() if '/' in mime_type else 'webm'
result = subprocess.run(
['ffmpeg', '-loglevel', 'error',
'-f', fmt, '-i', 'pipe:0',
'-ar', '16000', '-ac', '1', '-f', 's16le', 'pipe:1'],
input=audio_bytes,
capture_output=True,
)
if result.returncode != 0:
raise RuntimeError(f'ffmpeg PCM conversion failed: {result.stderr.decode()[:200]}')
return result.stdout
def _do_asr(audio_bytes: bytes, mime_type: str = 'audio/webm') -> str:
try:
import websocket as ws_module
except ImportError:
raise RuntimeError('websocket-client not installed — run: uv add websocket-client')
cfg = _get_cfg()
app_id = cfg['volc_app_id']
token = cfg['volc_access_token']
resource_id = cfg['volc_asr_resource']
if not app_id or not token:
raise ValueError('Volcengine credentials not configured — set them in JBOT Settings → API 配置')
pcm = _to_pcm(audio_bytes, mime_type)
if not pcm:
raise RuntimeError('PCM conversion produced empty audio')
result = {'text': '', 'error': None}
done_ev = threading.Event()
def on_open(ws):
config = json.dumps({
'user': {'uid': 'jbot-user'},
'audio': {'format': 'pcm', 'codec': 'raw', 'rate': 16000, 'bits': 16, 'channel': 1},
'request': {
'model_name': 'bigmodel',
'enable_itn': True,
'enable_punc': True,
'enable_ddc': False,
'result_type': 'full',
'end_window_size': 800,
},
}).encode()
ws.send(_asr_pack(_ASR_MT_FULL_CLIENT, _FLAG_NO_SEQ,
_SER_JSON, _CMP_GZIP, gzip.compress(config)),
ws_module.ABNF.OPCODE_BINARY)
CHUNK = 6400
pcm_data = pcm
for off in range(0, len(pcm_data), CHUNK):
chunk = pcm_data[off:off + CHUNK]
is_last = (off + CHUNK >= len(pcm_data))
ws.send(_asr_pack(
_ASR_MT_AUDIO_ONLY,
_FLAG_LAST_NO if is_last else _FLAG_NO_SEQ,
_SER_NONE, _CMP_GZIP, gzip.compress(chunk),
), ws_module.ABNF.OPCODE_BINARY)
def on_message(ws, data):
try:
parsed = _asr_unpack(data)
if parsed and parsed.get('result'):
t = parsed['result'].get('text', '')
if not t:
t = ''.join(
u['text'] for u in parsed['result'].get('utterances', [])
if u.get('definite')
)
if t:
result['text'] = t
except Exception as e:
logger.warning('[ASR] parse error: %s', e)
def on_close(ws, code, msg):
done_ev.set()
def on_error(ws, error):
err_str = str(error)
# opcode=8 is a normal WebSocket close frame — Volcengine server closes
# the connection after receiving the last audio chunk. Not a real error.
if 'opcode=8' in err_str or 'connection is already closed' in err_str.lower():
logger.debug('[ASR] server closed connection normally: %s', err_str)
done_ev.set()
return
result['error'] = err_str
done_ev.set()
wsa = ws_module.WebSocketApp(
'wss://openspeech.bytedance.com/api/v3/sauc/bigmodel',
header={
'X-Api-App-Key': app_id,
'X-Api-Access-Key': token,
'X-Api-Resource-Id': resource_id,
'X-Api-Connect-Id': str(uuid.uuid4()),
},
on_open=on_open,
on_message=on_message,
on_close=on_close,
on_error=on_error,
)
t = threading.Thread(target=wsa.run_forever, daemon=True)
t.start()
done_ev.wait(timeout=15)
if result['error']:
raise RuntimeError(result['error'])
return result['text']
@csrf_exempt
def asr_view(request):
if request.method != 'POST':
return JsonResponse({'error': 'POST required'}, status=405)
data = _json_body(request)
audio_b64 = data.get('audioBase64', '')
mime_type = data.get('format', 'audio/webm')
if not audio_b64:
return JsonResponse({'error': 'audioBase64 required'}, status=400)
try:
audio_bytes = base64.b64decode(audio_b64)
text = _do_asr(audio_bytes, mime_type)
return JsonResponse({'text': text or ''})
except Exception as e:
logger.error('[ASR] %s', e)
return JsonResponse({'error': str(e)}, status=500)
# ── Chat (OpenRouter LLM proxy) ───────────────────────────────────────────────
def _build_sys_prompt(settings, memories):
"""Shared system prompt builder for both backends."""
robot_name = settings.get('robotName', 'JBOT')
personality = settings.get('robotPersonality',
'你是一个友好、简洁、偶尔幽默的AI机器人助手。回复控制在80字以内。')
lang = settings.get('language', 'auto')
lang_hint = ''
if lang == 'zh-CN':
lang_hint = '\n请用中文回复。'
elif lang == 'en-US':
lang_hint = '\nPlease reply in English.'
sys_content = f'你叫 {robot_name}{personality}{lang_hint}'
if memories:
sys_content += '\n\n关于用户的长期记忆:\n' + '\n'.join(f'- {m}' for m in memories)
extra = settings.get('extraContext', '').strip()
if extra:
sys_content += f'\n\n用户背景信息:\n{extra}'
return sys_content
def _do_openrouter_chat(messages, sys_content, cfg):
"""Call OpenRouter /v1/chat/completions (non-streaming)."""
api_key = cfg['openrouter_api_key']
if not api_key:
raise ValueError('OPENROUTER_API_KEY not configured — set it in JBOT Settings → API 配置')
resp = requests.post(
'https://openrouter.ai/api/v1/chat/completions',
headers={
'Content-Type': 'application/json',
'Authorization': f'Bearer {api_key}',
'HTTP-Referer': 'https://go.junv.cc',
'X-Title': 'JBOT Robot Face',
},
json={
'model': cfg['llm_model'],
'messages': [{'role': 'system', 'content': sys_content}, *messages],
},
timeout=30,
)
resp.raise_for_status()
result = resp.json()
text = result.get('choices', [{}])[0].get('message', {}).get('content', '')
if not text:
raise RuntimeError(f'LLM returned no content: {result}')
return text, []
def _do_hermes_chat(messages, sys_content, cfg):
"""Call Hermes Agent /v1/chat/completions (streaming).
Returns (final_text, tool_calls) where tool_calls is a list of dicts
like {"tool": "terminal", "preview": "ls -la", "status": "start"}.
The stream is consumed completely before returning — callers get a clean
JSON response, not an SSE stream. Only the final assistant text is spoken
via TTS; tool-progress events are shown as status in the UI.
"""
base_url = cfg.get('hermes_url', 'http://192.168.1.10:8642').rstrip('/')
api_key = cfg.get('hermes_api_key', '')
headers = {'Content-Type': 'application/json'}
if api_key:
headers['Authorization'] = f'Bearer {api_key}'
payload = {
'model': 'hermes-agent',
'messages': [{'role': 'system', 'content': sys_content}, *messages],
'stream': True,
}
try:
resp = requests.post(
f'{base_url}/v1/chat/completions',
headers=headers,
json=payload,
stream=True,
timeout=120,
)
resp.raise_for_status()
except requests.exceptions.ConnectionError as e:
raise RuntimeError(f'Cannot connect to Hermes Agent at {base_url} — is it running? ({e})')
final_text = []
tool_calls = []
current_event = None
for raw_line in resp.iter_lines(decode_unicode=True):
if not raw_line:
# Blank line resets current event type (SSE separator)
current_event = None
continue
if raw_line.startswith('event:'):
current_event = raw_line[6:].strip()
continue
if raw_line.startswith('data:'):
data_str = raw_line[5:].strip()
if data_str == '[DONE]':
break
try:
obj = json.loads(data_str)
except Exception:
continue
if current_event == 'hermes.tool.progress':
tool_name = obj.get('tool', '')
preview = obj.get('preview', '')
status = obj.get('status', '')
if tool_name and status == 'start':
tool_calls.append({'tool': tool_name, 'preview': preview})
else:
delta = (obj.get('choices') or [{}])[0].get('delta', {})
content = delta.get('content') or ''
if content:
final_text.append(content)
text = ''.join(final_text).strip()
if not text:
raise RuntimeError('Hermes Agent returned no text response')
return text, tool_calls
@csrf_exempt
def chat_view(request):
if request.method != 'POST':
return JsonResponse({'error': 'POST required'}, status=405)
cfg = _get_cfg()
data = _json_body(request)
messages = data.get('messages', [])
memories = data.get('memories', [])
settings = data.get('settings', {})
mode = cfg.get('backend_mode', 'openrouter')
# In Hermes mode, don't inject local memories — Hermes manages its own memory
sys_content = _build_sys_prompt(settings, [] if mode == 'hermes' else memories)
try:
if mode == 'hermes':
text, tool_calls = _do_hermes_chat(messages, sys_content, cfg)
return JsonResponse({'text': text, 'toolCalls': tool_calls, 'backend': 'hermes'})
else:
text, tool_calls = _do_openrouter_chat(messages, sys_content, cfg)
return JsonResponse({'text': text, 'backend': 'openrouter'})
except Exception as e:
logger.error('[CHAT/%s] %s', mode, e)
return JsonResponse({'error': str(e)}, status=500)
# ── Memory extraction ─────────────────────────────────────────────────────────
@csrf_exempt
def memory_extract_view(request):
if request.method != 'POST':
return JsonResponse({'error': 'POST required'}, status=405)
cfg = _get_cfg()
api_key = cfg['openrouter_api_key']
if not api_key:
return JsonResponse({'facts': []})
data = _json_body(request)
user_text = data.get('userText', '')
bot_text = data.get('botText', '')
if not user_text or not bot_text:
return JsonResponse({'facts': []})
model = cfg['llm_model']
prompt = (
'You are a memory extraction assistant for an AI robot named JBOT.\n\n'
f'Conversation exchange:\nUser: "{user_text}"\nJBOT: "{bot_text}"\n\n'
'Extract 03 short factual statements about the USER worth remembering long-term '
'(e.g. name, profession, preferences, ongoing projects, location). '
'Max 20 words each. Only lasting facts, not one-time remarks. '
'Reply with a JSON array of strings, e.g. ["User is a software engineer", ...]. '
'If nothing worth remembering, reply with [].'
)
try:
resp = requests.post(
'https://openrouter.ai/api/v1/chat/completions',
headers={
'Content-Type': 'application/json',
'Authorization': f'Bearer {api_key}',
'HTTP-Referer': 'https://go.junv.cc',
'X-Title': 'JBOT Memory',
},
json={
'model': model,
'messages': [{'role': 'user', 'content': prompt}],
},
timeout=15,
)
resp.raise_for_status()
raw = resp.json().get('choices', [{}])[0].get('message', {}).get('content', '[]').strip()
if raw.startswith('```'):
raw = raw.split('\n', 1)[-1].rsplit('```', 1)[0].strip()
facts = json.loads(raw)
if not isinstance(facts, list):
facts = []
return JsonResponse({'facts': [str(f) for f in facts[:3]]})
except Exception as e:
logger.warning('[MEMORY] extraction failed: %s', e)
return JsonResponse({'facts': []})
# ── Public config flags ────────────────────────────────────────────────────────
def config_view(request):
cfg = _get_cfg()
return JsonResponse({
'volcengine': bool(cfg['volc_app_id'] and cfg['volc_access_token']),
'llm': bool(cfg['openrouter_api_key']),
'ttsVoice': cfg['volc_tts_voice'],
'backendMode': cfg.get('backend_mode', 'openrouter'),
})