mirror of
https://github.com/wahyd4/links.git
synced 2026-08-08 21:04:53 +10:00
junvandCopilot
a87b4643fd
- Add jbot Django app (jbot/__init__.py, apps.py, urls.py, api_urls.py) - Python backend (jbot/views.py): TTS 2.0, ASR 1.0 BigASR, LLM/memory via OpenRouter - React SPA frontend at /jbot/ (static_src/jbot/): RobotFace, ChatPanel, expressions, voices, push-to-talk, history, memory, settings pages - Vite entry jbot: static_src/jbot/main.jsx → dist/jbot.js + dist/jbot.css - react-router-dom added; BrowserRouter basename=/jbot for SPA routing - core/settings.py: added jbot to INSTALLED_APPS - core/urls.py: /jbot/ + /api/jbot/ URL includes - pyproject.toml: websocket-client>=1.9.0 for BigASR binary WS protocol - Dockerfile: ffmpeg added to production apt-get for WebM→PCM audio conversion - k8s/manifest.yaml: VOLC_APP_ID, VOLC_ACCESS_TOKEN, OPENROUTER_API_KEY env vars via jbot-credentials Secret; LLM_MODEL, VOLC_TTS_VOICE, VOLC_ASR_RESOURCE defaults Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
443 lines
16 KiB
Python
443 lines
16 KiB
Python
"""
|
||
JBOT backend views — Python port of the Node.js server/index.js
|
||
Endpoints:
|
||
GET /jbot/ → SPA shell (index.html)
|
||
POST /api/jbot/chat/ → OpenRouter LLM proxy
|
||
POST /api/jbot/tts/ → Volcengine seed-tts-2.0
|
||
POST /api/jbot/asr/ → Volcengine BigASR 1.0 (WebSocket binary protocol)
|
||
POST /api/jbot/memory/extract/ → background memory extraction via LLM
|
||
GET /api/jbot/config/ → public config flags
|
||
"""
|
||
import base64
|
||
import gzip
|
||
import json
|
||
import logging
|
||
import os
|
||
import struct
|
||
import subprocess
|
||
import threading
|
||
import uuid
|
||
|
||
import requests
|
||
from django.http import JsonResponse
|
||
from django.utils.decorators import method_decorator
|
||
from django.views import View
|
||
from django.views.decorators.csrf import csrf_exempt
|
||
from django.views.generic import TemplateView
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
# ── SPA shell ────────────────────────────────────────────────────────────────
|
||
|
||
class JbotIndexView(TemplateView):
|
||
template_name = 'jbot/index.html'
|
||
|
||
|
||
# ── Helper: require JSON POST ─────────────────────────────────────────────────
|
||
|
||
def _json_body(request):
|
||
try:
|
||
return json.loads(request.body)
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
# ── TTS 2.0 (Volcengine seed-tts-2.0 HTTP streaming NDJSON) ─────────────────
|
||
|
||
def _do_tts(text, voice=None):
|
||
app_id = os.environ.get('VOLC_APP_ID', '')
|
||
token = os.environ.get('VOLC_ACCESS_TOKEN', '')
|
||
speaker = voice or os.environ.get('VOLC_TTS_VOICE', 'zh_female_vv_uranus_bigtts')
|
||
resource_id = os.environ.get('VOLC_TTS_RESOURCE', 'seed-tts-2.0')
|
||
|
||
if not app_id or not token:
|
||
raise ValueError('Volcengine credentials not configured (VOLC_APP_ID / VOLC_ACCESS_TOKEN)')
|
||
|
||
resp = requests.post(
|
||
'https://openspeech.bytedance.com/api/v3/tts/unidirectional',
|
||
headers={
|
||
'Content-Type': 'application/json',
|
||
'X-Api-App-Id': app_id,
|
||
'X-Api-Access-Key': token,
|
||
'X-Api-Resource-Id': resource_id,
|
||
'X-Api-Request-Id': str(uuid.uuid4()),
|
||
},
|
||
json={
|
||
'user': {'uid': 'jbot-user'},
|
||
'req_params': {
|
||
'text': text,
|
||
'speaker': speaker,
|
||
'audio_params': {'format': 'mp3', 'sample_rate': 24000, 'speech_rate': 0},
|
||
},
|
||
},
|
||
stream=True,
|
||
timeout=30,
|
||
)
|
||
|
||
if not resp.ok:
|
||
raise RuntimeError(f'TTS HTTP {resp.status_code}: {resp.text[:200]}')
|
||
|
||
audio_chunks = []
|
||
for line in resp.iter_lines():
|
||
if not line:
|
||
continue
|
||
try:
|
||
obj = json.loads(line)
|
||
except Exception:
|
||
continue
|
||
if obj.get('code') == 0 and obj.get('data'):
|
||
audio_chunks.append(base64.b64decode(obj['data']))
|
||
elif obj.get('code') == 20000000:
|
||
break
|
||
elif obj.get('code', 0) != 0:
|
||
raise RuntimeError(f"TTS error code {obj.get('code')}: {obj.get('message', '')}")
|
||
|
||
if not audio_chunks:
|
||
raise RuntimeError('TTS returned no audio data')
|
||
|
||
return b''.join(audio_chunks)
|
||
|
||
|
||
@csrf_exempt
|
||
def tts_view(request):
|
||
if request.method != 'POST':
|
||
return JsonResponse({'error': 'POST required'}, status=405)
|
||
data = _json_body(request)
|
||
text = data.get('text', '').strip()
|
||
voice = data.get('voice', '')
|
||
if not text:
|
||
return JsonResponse({'error': 'text required'}, status=400)
|
||
try:
|
||
audio_bytes = _do_tts(text, voice or None)
|
||
return JsonResponse({
|
||
'audioBase64': base64.b64encode(audio_bytes).decode(),
|
||
'format': 'mp3',
|
||
})
|
||
except Exception as e:
|
||
logger.error('[TTS] %s', e)
|
||
return JsonResponse({'error': str(e)}, status=500)
|
||
|
||
|
||
# ── ASR 1.0 (Volcengine BigASR WebSocket binary protocol) ────────────────────
|
||
|
||
# Message type constants
|
||
_ASR_MT_FULL_CLIENT = 1
|
||
_ASR_MT_AUDIO_ONLY = 2
|
||
_ASR_MT_FULL_SERVER = 9
|
||
_ASR_MT_ERROR = 15
|
||
|
||
# Flag constants
|
||
_FLAG_NO_SEQ = 0
|
||
_FLAG_POS_SEQ = 1
|
||
_FLAG_LAST_NO = 2
|
||
_FLAG_NEG_SEQ = 3
|
||
|
||
# Serialization / compression
|
||
_SER_NONE = 0
|
||
_SER_JSON = 1
|
||
_CMP_NONE = 0
|
||
_CMP_GZIP = 1
|
||
|
||
|
||
def _asr_pack(msg_type, flag, serialization=_SER_NONE, compression=_CMP_NONE,
|
||
payload=b'', sequence=None):
|
||
header = bytes([0x11, (msg_type << 4) | flag, (serialization << 4) | compression, 0x00])
|
||
parts = [header]
|
||
if flag in (_FLAG_POS_SEQ, _FLAG_NEG_SEQ):
|
||
parts.append(struct.pack('>i', sequence or 0))
|
||
parts.append(struct.pack('>I', len(payload)))
|
||
parts.append(payload)
|
||
return b''.join(parts)
|
||
|
||
|
||
def _asr_unpack(data: bytes):
|
||
header_bytes = (data[0] & 0x0F) * 4
|
||
msg_type = (data[1] >> 4) & 0x0F
|
||
flag = data[1] & 0x0F
|
||
compression = data[2] & 0x0F
|
||
|
||
off = header_bytes
|
||
if flag in (_FLAG_POS_SEQ, _FLAG_NEG_SEQ):
|
||
off += 4
|
||
|
||
if msg_type == _ASR_MT_ERROR:
|
||
code = struct.unpack_from('>I', data, off)[0]; off += 4
|
||
msg_len = struct.unpack_from('>I', data, off)[0]; off += 4
|
||
msg = data[off:off + msg_len].decode('utf-8', errors='replace')
|
||
raise RuntimeError(f'ASR error {code}: {msg}')
|
||
|
||
payload = b''
|
||
if off + 4 <= len(data):
|
||
p_len = struct.unpack_from('>I', data, off)[0]; off += 4
|
||
if p_len > 0:
|
||
payload = data[off:off + p_len]
|
||
|
||
if compression == _CMP_GZIP and payload:
|
||
payload = gzip.decompress(payload)
|
||
|
||
try:
|
||
return json.loads(payload.decode('utf-8'))
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _to_pcm(audio_bytes: bytes, mime_type: str = 'audio/webm') -> bytes:
|
||
"""Convert browser audio (webm/ogg/mp4) to PCM 16kHz 16-bit mono via ffmpeg."""
|
||
fmt = mime_type.split('/')[1].split(';')[0].strip() if '/' in mime_type else 'webm'
|
||
result = subprocess.run(
|
||
['ffmpeg', '-loglevel', 'error',
|
||
'-f', fmt, '-i', 'pipe:0',
|
||
'-ar', '16000', '-ac', '1', '-f', 's16le', 'pipe:1'],
|
||
input=audio_bytes,
|
||
capture_output=True,
|
||
)
|
||
if result.returncode != 0:
|
||
raise RuntimeError(f'ffmpeg PCM conversion failed: {result.stderr.decode()[:200]}')
|
||
return result.stdout
|
||
|
||
|
||
def _do_asr(audio_bytes: bytes, mime_type: str = 'audio/webm') -> str:
|
||
"""Transcribe audio using Volcengine BigASR 1.0 WebSocket binary protocol."""
|
||
try:
|
||
import websocket as ws_module
|
||
except ImportError:
|
||
raise RuntimeError('websocket-client not installed — run: uv add websocket-client')
|
||
|
||
app_id = os.environ.get('VOLC_APP_ID', '')
|
||
token = os.environ.get('VOLC_ACCESS_TOKEN', '')
|
||
resource_id = os.environ.get('VOLC_ASR_RESOURCE', 'volc.bigasr.sauc.duration')
|
||
|
||
if not app_id or not token:
|
||
raise ValueError('Volcengine credentials not configured')
|
||
|
||
pcm = _to_pcm(audio_bytes, mime_type)
|
||
if not pcm:
|
||
raise RuntimeError('PCM conversion produced empty audio')
|
||
|
||
result = {'text': '', 'error': None}
|
||
done_ev = threading.Event()
|
||
|
||
def on_open(ws):
|
||
config = json.dumps({
|
||
'user': {'uid': 'jbot-user'},
|
||
'audio': {'format': 'pcm', 'codec': 'raw', 'rate': 16000, 'bits': 16, 'channel': 1},
|
||
'request': {
|
||
'model_name': 'bigmodel',
|
||
'enable_itn': True,
|
||
'enable_punc': True,
|
||
'enable_ddc': False,
|
||
'result_type': 'full',
|
||
'end_window_size': 800,
|
||
},
|
||
}).encode()
|
||
ws.send_binary(_asr_pack(_ASR_MT_FULL_CLIENT, _FLAG_NO_SEQ,
|
||
_SER_JSON, _CMP_GZIP, gzip.compress(config)))
|
||
|
||
CHUNK = 6400
|
||
pcm_data = pcm # captured from outer scope
|
||
for off in range(0, len(pcm_data), CHUNK):
|
||
chunk = pcm_data[off:off + CHUNK]
|
||
is_last = (off + CHUNK >= len(pcm_data))
|
||
ws.send_binary(_asr_pack(
|
||
_ASR_MT_AUDIO_ONLY,
|
||
_FLAG_LAST_NO if is_last else _FLAG_NO_SEQ,
|
||
_SER_NONE, _CMP_GZIP, gzip.compress(chunk),
|
||
))
|
||
|
||
def on_message(ws, data):
|
||
try:
|
||
parsed = _asr_unpack(data)
|
||
if parsed and parsed.get('result'):
|
||
t = parsed['result'].get('text', '')
|
||
if not t:
|
||
t = ''.join(
|
||
u['text'] for u in parsed['result'].get('utterances', [])
|
||
if u.get('definite')
|
||
)
|
||
if t:
|
||
result['text'] = t
|
||
except Exception as e:
|
||
logger.warning('[ASR] parse error: %s', e)
|
||
|
||
def on_close(ws, code, msg):
|
||
done_ev.set()
|
||
|
||
def on_error(ws, error):
|
||
result['error'] = str(error)
|
||
done_ev.set()
|
||
|
||
wsa = ws_module.WebSocketApp(
|
||
'wss://openspeech.bytedance.com/api/v3/sauc/bigmodel',
|
||
header={
|
||
'X-Api-App-Key': app_id,
|
||
'X-Api-Access-Key': token,
|
||
'X-Api-Resource-Id': resource_id,
|
||
'X-Api-Connect-Id': str(uuid.uuid4()),
|
||
},
|
||
on_open=on_open,
|
||
on_message=on_message,
|
||
on_close=on_close,
|
||
on_error=on_error,
|
||
)
|
||
|
||
t = threading.Thread(target=wsa.run_forever, daemon=True)
|
||
t.start()
|
||
done_ev.wait(timeout=15)
|
||
|
||
if result['error']:
|
||
raise RuntimeError(result['error'])
|
||
|
||
return result['text']
|
||
|
||
|
||
@csrf_exempt
|
||
def asr_view(request):
|
||
if request.method != 'POST':
|
||
return JsonResponse({'error': 'POST required'}, status=405)
|
||
data = _json_body(request)
|
||
audio_b64 = data.get('audioBase64', '')
|
||
mime_type = data.get('format', 'audio/webm')
|
||
if not audio_b64:
|
||
return JsonResponse({'error': 'audioBase64 required'}, status=400)
|
||
try:
|
||
audio_bytes = base64.b64decode(audio_b64)
|
||
text = _do_asr(audio_bytes, mime_type)
|
||
return JsonResponse({'text': text or ''})
|
||
except Exception as e:
|
||
logger.error('[ASR] %s', e)
|
||
return JsonResponse({'error': str(e)}, status=500)
|
||
|
||
|
||
# ── Chat (OpenRouter LLM proxy) ───────────────────────────────────────────────
|
||
|
||
@csrf_exempt
|
||
def chat_view(request):
|
||
if request.method != 'POST':
|
||
return JsonResponse({'error': 'POST required'}, status=405)
|
||
|
||
api_key = os.environ.get('OPENROUTER_API_KEY', '')
|
||
if not api_key:
|
||
return JsonResponse({'error': 'OPENROUTER_API_KEY not configured'}, status=503)
|
||
|
||
data = _json_body(request)
|
||
messages = data.get('messages', [])
|
||
memories = data.get('memories', [])
|
||
settings = data.get('settings', {})
|
||
model = os.environ.get('LLM_MODEL', 'deepseek/deepseek-chat')
|
||
|
||
robot_name = settings.get('robotName', 'JBOT')
|
||
personality = settings.get('robotPersonality',
|
||
'你是一个友好、简洁、偶尔幽默的AI机器人助手。回复控制在80字以内。')
|
||
lang = settings.get('language', 'auto')
|
||
lang_hint = ''
|
||
if lang == 'zh-CN':
|
||
lang_hint = '\n请用中文回复。'
|
||
elif lang == 'en-US':
|
||
lang_hint = '\nPlease reply in English.'
|
||
|
||
sys_content = f'你叫 {robot_name}。{personality}{lang_hint}'
|
||
|
||
if memories:
|
||
sys_content += '\n\n关于用户的长期记忆：\n' + '\n'.join(f'- {m}' for m in memories)
|
||
|
||
extra = settings.get('extraContext', '').strip()
|
||
if extra:
|
||
sys_content += f'\n\n用户背景信息：\n{extra}'
|
||
|
||
try:
|
||
resp = requests.post(
|
||
'https://openrouter.ai/api/v1/chat/completions',
|
||
headers={
|
||
'Content-Type': 'application/json',
|
||
'Authorization': f'Bearer {api_key}',
|
||
'HTTP-Referer': 'https://go.junv.cc',
|
||
'X-Title': 'JBOT Robot Face',
|
||
},
|
||
json={
|
||
'model': model,
|
||
'messages': [{'role': 'system', 'content': sys_content}, *messages],
|
||
},
|
||
timeout=30,
|
||
)
|
||
resp.raise_for_status()
|
||
result = resp.json()
|
||
text = result.get('choices', [{}])[0].get('message', {}).get('content', '')
|
||
if not text:
|
||
logger.error('[CHAT] LLM returned no content: %s', result)
|
||
return JsonResponse({'error': 'LLM returned no content'}, status=500)
|
||
return JsonResponse({'text': text})
|
||
except Exception as e:
|
||
logger.error('[CHAT] %s', e)
|
||
return JsonResponse({'error': str(e)}, status=500)
|
||
|
||
|
||
# ── Memory extraction ─────────────────────────────────────────────────────────
|
||
|
||
@csrf_exempt
|
||
def memory_extract_view(request):
|
||
if request.method != 'POST':
|
||
return JsonResponse({'error': 'POST required'}, status=405)
|
||
|
||
api_key = os.environ.get('OPENROUTER_API_KEY', '')
|
||
if not api_key:
|
||
return JsonResponse({'facts': []})
|
||
|
||
data = _json_body(request)
|
||
user_text = data.get('userText', '')
|
||
bot_text = data.get('botText', '')
|
||
if not user_text or not bot_text:
|
||
return JsonResponse({'facts': []})
|
||
|
||
model = os.environ.get('LLM_MODEL', 'deepseek/deepseek-chat')
|
||
prompt = (
|
||
'You are a memory extraction assistant for an AI robot named JBOT.\n\n'
|
||
f'Conversation exchange:\nUser: "{user_text}"\nJBOT: "{bot_text}"\n\n'
|
||
'Extract 0–3 short factual statements about the USER worth remembering long-term '
|
||
'(e.g. name, profession, preferences, ongoing projects, location). '
|
||
'Max 20 words each. Only lasting facts, not one-time remarks. '
|
||
'Reply with a JSON array of strings, e.g. ["User is a software engineer", ...]. '
|
||
'If nothing worth remembering, reply with [].'
|
||
)
|
||
|
||
try:
|
||
resp = requests.post(
|
||
'https://openrouter.ai/api/v1/chat/completions',
|
||
headers={
|
||
'Content-Type': 'application/json',
|
||
'Authorization': f'Bearer {api_key}',
|
||
'HTTP-Referer': 'https://go.junv.cc',
|
||
'X-Title': 'JBOT Memory',
|
||
},
|
||
json={
|
||
'model': model,
|
||
'messages': [{'role': 'user', 'content': prompt}],
|
||
},
|
||
timeout=15,
|
||
)
|
||
resp.raise_for_status()
|
||
raw = resp.json().get('choices', [{}])[0].get('message', {}).get('content', '[]')
|
||
raw = raw.strip()
|
||
# strip ```json fences if present
|
||
if raw.startswith('```'):
|
||
raw = raw.split('\n', 1)[-1].rsplit('```', 1)[0].strip()
|
||
facts = json.loads(raw)
|
||
if not isinstance(facts, list):
|
||
facts = []
|
||
return JsonResponse({'facts': [str(f) for f in facts[:3]]})
|
||
except Exception as e:
|
||
logger.warning('[MEMORY] extraction failed: %s', e)
|
||
return JsonResponse({'facts': []})
|
||
|
||
|
||
# ── Config ────────────────────────────────────────────────────────────────────
|
||
|
||
def config_view(request):
|
||
has_volc = bool(os.environ.get('VOLC_APP_ID') and os.environ.get('VOLC_ACCESS_TOKEN'))
|
||
has_llm = bool(os.environ.get('OPENROUTER_API_KEY'))
|
||
return JsonResponse({
|
||
'volcengine': has_volc,
|
||
'llm': has_llm,
|
||
'ttsVoice': os.environ.get('VOLC_TTS_VOICE', 'zh_female_vv_uranus_bigtts'),
|
||
})
|