feat: AI 控制台升级为顶级 Tab——DeepSeek 风格聊天界面(气泡+流式渲染+思考折叠),实时 MCP 步骤卡片(工具/参数/截图缩略),SSE 流式输出(delta/step/done/error 事件),配置模态(模型/Key/设备前端可配);工具页旧子栏移除

This commit is contained in:
2026-09-04 13:09:34 +08:00
parent 25ef7e7746
commit e24458a085
6 changed files with 442 additions and 198 deletions
+110 -54
View File
@@ -1,11 +1,13 @@
"""Agent 编排层:OpenAI 兼容模型(DeepSeek 等)经 MCP 工具控制手机。
工作流:
1. 启动时从 MCP Server 拉工具列表 → 转 OpenAI function schema
2. run(prompt):循环 chat/completions
- 模型返回 tool_calls → 依次执行(经 MCP)→ 结果回喂
- de_screenshot 的返回图像转为 image_url 追加为下一轮 user 消息(多模态看图)
- 无 tool_calls → 返回最终文本
支持两种运行模式:
- run_stream():流式(SSE 逐 token + 工具调用实时回调)——Web AI 控制台用
- run():非流式收集结果——CLI 用(内部调 run_stream)
流式细节(OpenAI 兼容):
- content/reasoning_content 增量逐 chunk 回调(kind 区分)
- tool_calls 分片累积(arguments 按 index 拼接),流结束后统一执行
- 截图(de_screenshot)图像转 image_url 追加下一轮,同时 on_tool 回调带缩略
"""
import base64
import json
@@ -26,9 +28,10 @@ SYSTEM_PROMPT = """你是手机自动化控制助手。你通过工具实时操
1. 先 de_list_devices 确定目标设备(在线才可操作)
2. 观察屏幕:先 de_screenshot 获取截图(图像会随后给你),基于截图理解当前界面
3. 操作:de_tap/de_swipe 的坐标必须与最近一次 de_screenshot 图像一致(直接看图给坐标,服务器自动换算)
4. 每次关键操作后再次 de_screenshot 验证结果,直到完成用户目标
5. 完成或失败时用中文总结:做了什么、当前状态、需要用户注意的事项
6. 设备不可用/操作失败时如实报告错误,不要臆测成功
4. 元素操作优先:能用 de_ui_tree/de_tap_element(text/id/desc 定位)就不用裸坐标
5. 每次关键操作后再次 de_screenshot 验证结果,直到完成用户目标
6. 完成或失败时用中文总结:做了什么、当前状态、需要用户注意的事项
7. 设备不可用/操作失败时如实报告错误,不要臆测成功
可用工具清单将由系统提供。"""
@@ -37,9 +40,13 @@ class Agent:
def __init__(self, settings: AgentSettings = None):
self.s = settings or S
self.tools_schema = [] # OpenAI function schema
self._tool_exec = {} # name -> callable
self.messages = []
self.on_step = None # 可选回调 fn(step_dict),web 展示进度用
# 回调(Web 展示用,均可选):
# on_delta(text, kind) kind: content | reasoning —— 流式文本增量
# on_tool(step) step: {tool, args, result, image} —— 工具调用完成
self.on_delta = None
self.on_tool = None
self._mcp = None
# ---------- MCP 工具桥 ----------
async def _load_tools(self):
@@ -52,37 +59,49 @@ class Agent:
for t in tools:
# MCP SDK v2 改名 input_schema,兼容新旧字段
schema = getattr(t, "input_schema", None) or getattr(t, "inputSchema", {})
# fastmcp Tool 属性兼容:name/description/inputSchema
name = getattr(t, "name", "")
desc = getattr(t, "description", "") or ""
self.tools_schema.append({
"type": "function",
"function": {"name": name, "description": desc,
"parameters": schema}})
self._tool_exec[name] = t
_log.info("MCP 工具已加载: %s", [s["function"]["name"] for s in self.tools_schema])
async def close(self):
if getattr(self, "_mcp", None):
await self._mcp.__aexit__(None, None, None)
if self._mcp:
try:
await self._mcp.__aexit__(None, None, None)
except Exception:
pass
# ---------- 模型调用 ----------
async def _chat(self):
"""调用 OpenAI 兼容 chat/completions,返回完整 response JSON。"""
# ---------- 模型调用(流式) ----------
async def _chat_stream(self):
"""流式 chat/completions:逐 chunk 产出 JSON(async generator)。"""
body = {
"model": self.s.model,
"messages": self.messages,
"tools": self.tools_schema if self.tools_schema else None,
"max_tokens": 4096,
"stream": True,
}
headers = {"Authorization": f"Bearer {self.s.api_key}",
"Content-Type": "application/json"}
url = f"{self.s.api_base.rstrip('/')}/chat/completions"
async with httpx.AsyncClient(timeout=self.s.request_timeout) as client:
r = await client.post(f"{self.s.api_base.rstrip('/')}/chat/completions",
json=body, headers=headers)
if r.status_code != 200:
raise RuntimeError(f"模型 API HTTP {r.status_code}: {r.text[:300]}")
return r.json()
async with client.stream("POST", url, json=body, headers=headers) as r:
if r.status_code != 200:
text = (await r.aread()).decode(errors="replace")
raise RuntimeError(f"模型 API HTTP {r.status_code}: {text[:300]}")
async for line in r.aiter_lines():
if not line.startswith("data:"):
continue
data = line[5:].strip()
if data == "[DONE]":
break
try:
yield json.loads(data)
except json.JSONDecodeError:
continue
# ---------- 工具执行 ----------
async def _execute_tool(self, name, arguments):
@@ -94,28 +113,33 @@ class Agent:
data = getattr(result, "data", result)
except Exception as e:
return {"ok": False, "error": f"工具执行失败: {e}"}, None
# de_screenshot:图像分离(作为 image_url 追加给模型看)
# de_screenshot:图像分离(作为 image_url 追加给模型看 + on_tool 缩略展示)
image_b64 = None
text_result = data
if name == "de_screenshot" and isinstance(data, dict) and data.get("ok"):
img = (data.get("data") or {}).get("image") or {}
if img.get("data"):
text_result = {k: v for k, v in (data.get("data") or {}).items()
if k != "image"}
image_b64 = img["data"]
if self.on_step:
if self.on_tool:
try:
self.on_step({"tool": name, "args": args,
"result": text_result if image_b64 else data,
"image_b64": image_b64})
self.on_tool({"tool": name, "args": args,
"result": text_result, "image": image_b64})
except Exception:
pass
if image_b64:
return text_result, image_b64
return data, None
return text_result, image_b64
# ---------- 主循环 ----------
async def run(self, prompt: str, serial: str = "") -> str:
"""执行一轮指令,返回最终回答文本。"""
# ---------- 主循环(流式) ----------
async def run_stream(self, prompt: str, serial: str = "",
on_delta=None, on_tool=None):
"""流式执行一轮指令,返回最终完整文本。
on_delta(text, kind):content/reasoning 文本增量(实时推给前端)
on_tool(step):工具调用完成(实时显示 MCP 步骤)
"""
self.on_delta = on_delta
self.on_tool = on_tool
target = serial or self.s.default_serial
sys_txt = SYSTEM_PROMPT
if target:
@@ -123,28 +147,55 @@ class Agent:
self.messages = [{"role": "system", "content": sys_txt},
{"role": "user", "content": prompt}]
for step in range(self.s.max_steps):
resp = await self._chat()
choice = (resp.get("choices") or [{}])[0]
msg = choice.get("message") or {}
# 1) 工具调用
tool_calls = msg.get("tool_calls")
if tool_calls:
self.messages.append({
"role": "assistant",
"content": msg.get("content") or "",
"tool_calls": tool_calls})
for tc in tool_calls:
for _step in range(self.s.max_steps):
content_parts = []
tool_acc = {} # index -> {id, name, args}
has_tool = False
async for chunk in self._chat_stream():
choice = (chunk.get("choices") or [{}])[0]
delta = choice.get("delta") or {}
text = delta.get("content")
if text:
content_parts.append(text)
if on_delta:
on_delta(text, "content")
rtext = delta.get("reasoning_content")
if rtext:
if on_delta:
on_delta(rtext, "reasoning")
for tc in delta.get("tool_calls") or []:
has_tool = True
idx = tc.get("index", 0)
acc = tool_acc.setdefault(idx, {"id": "", "name": "", "args": ""})
if tc.get("id"):
acc["id"] = tc["id"]
fn = tc.get("function") or {}
name = fn.get("name", "")
if fn.get("name"):
acc["name"] += fn["name"]
if fn.get("arguments"):
acc["args"] += fn["arguments"]
full_content = "".join(content_parts)
if has_tool:
# 组装 assistant 消息(含 tool_calls)并执行工具
tcs = []
for idx in sorted(tool_acc):
acc = tool_acc[idx]
tcs.append({"id": acc["id"] or f"call_{idx}",
"type": "function",
"function": {"name": acc["name"],
"arguments": acc["args"]}})
self.messages.append({"role": "assistant",
"content": full_content,
"tool_calls": tcs})
for tc in tcs:
fn = tc["function"]
text_result, image_b64 = await self._execute_tool(
name, fn.get("arguments", "{}"))
fn["name"], fn["arguments"])
self.messages.append({
"role": "tool",
"tool_call_id": tc.get("id", ""),
"role": "tool", "tool_call_id": tc["id"],
"content": json.dumps(text_result, ensure_ascii=False)[:4000]})
# 截图图像:作为下一轮 user 图像内容(OpenAI 协议 tool 结果只能文本)
if image_b64:
self.messages.append({
"role": "user",
@@ -155,7 +206,12 @@ class Agent:
f"data:image/jpeg;base64,{image_b64}"}}]})
continue
# 2) 最终回答
return msg.get("content") or "(模型无输出)"
# 无工具调用:本轮即最终回答
return full_content
return "(达到最大步骤数未完成,请检查操作是否卡在循环)"
# ---------- 非流式(CLI) ----------
async def run(self, prompt: str, serial: str = "") -> str:
"""非流式执行,返回最终文本(CLI 用,内部走流式收集)。"""
return await self.run_stream(prompt, serial)
+198
View File
@@ -0,0 +1,198 @@
// AI 控制台(顶级 Tab):DeepSeek 风格聊天 + 流式输出 + 实时 MCP 步骤
let _agentBusy = false;
let _agentStream = null; // EventSource
let _agentCfgLoaded = false;
// ================== 初始化 ==================
function initAgentChat(){
if(_agentCfgLoaded)return;
_agentCfgLoaded = true;
loadAgentConfig();
const chat = document.getElementById('agent-chat');
if(chat && !chat.children.length){
chat.innerHTML = '<div class="agent-empty">👋 给 AI 下达指令,它将通过截图观察手机并执行操作。<br>'
+ '例如:「打开抖音搜索奚学东,告诉我第一个视频的标题」<br>'
+ '点击右上角 ⚙ 配置模型与 API Key。</div>';
}
// 输入框快捷键
const inp = document.getElementById('agent-input');
inp.addEventListener('keydown', ev=>{
if(ev.key==='Enter' && !ev.shiftKey){
ev.preventDefault();
if(!_agentBusy) sendAgentMsg();
}
});
}
// ================== 配置 ==================
function loadAgentConfig(){
apiGet('/api/agent/config').then(r=>{
if(!r||!r.ok)return;
const tag = document.getElementById('agent-model-tag');
if(tag && r.model) tag.textContent = r.model;
});
}
function openAgentConfig(){
apiGet('/api/agent/config').then(r=>{
if(!r||!r.ok)return;
document.getElementById('acfg-base').value = r.api_base || 'https://api.deepseek.com';
document.getElementById('acfg-model').value = r.model || '';
document.getElementById('acfg-serial').value = r.default_serial || '';
document.getElementById('acfg-key-hint').textContent =
r.api_key_masked ? ('已配置 ' + r.api_key_masked) : '未配置';
document.getElementById('acfg-key').value = '';
document.getElementById('agent-cfg-overlay').style.display = 'flex';
});
}
function closeAgentConfig(){
document.getElementById('agent-cfg-overlay').style.display = 'none';
}
function saveAgentConfig(){
const body = {
api_base: document.getElementById('acfg-base').value.trim(),
model: document.getElementById('acfg-model').value.trim(),
default_serial: document.getElementById('acfg-serial').value.trim(),
};
const key = document.getElementById('acfg-key').value.trim();
if(key) body.api_key = key;
apiPost('/api/agent/config', body).then(r=>{
if(r&&r.ok){
showToast('配置已保存','success');
closeAgentConfig();
loadAgentConfig();
}else showToast('保存失败: ' + ((r&&r.error)||''),'error');
});
}
// ================== 聊天渲染 ==================
function addUserMsg(text){
const chat = document.getElementById('agent-chat');
clearEmpty();
const div = document.createElement('div');
div.className = 'agent-msg user';
div.textContent = text;
chat.appendChild(div);
scrollChat();
}
function newAssistantMsg(){
const chat = document.getElementById('agent-chat');
clearEmpty();
const div = document.createElement('div');
div.className = 'agent-msg assistant';
div.innerHTML = '<div class="agent-toolcards"></div><div class="agent-text"></div>';
chat.appendChild(div);
scrollChat();
return div;
}
function clearEmpty(){
const empty = document.querySelector('#agent-chat .agent-empty');
if(empty) empty.remove();
}
function scrollChat(){
const chat = document.getElementById('agent-chat');
chat.scrollTop = chat.scrollHeight;
}
function clearAgentChat(){
document.getElementById('agent-chat').innerHTML =
'<div class="agent-empty">对话已清空。给 AI 下达新指令…</div>';
}
// ================== 发送与流式接收 ==================
function sendAgentMsg(){
if(_agentBusy){showToast('上一轮还在运行','error');return;}
const inp = document.getElementById('agent-input');
const text = inp.value.trim();
if(!text){showToast('请输入指令','error');return;}
inp.value = '';
addUserMsg(text);
apiPost('/api/agent/run', {prompt: text}).then(r=>{
if(!r||!r.ok){
const msg = r ? (r.error||'启动失败') : '请求失败';
const div = newAssistantMsg();
div.querySelector('.agent-text').textContent = '⚠ ' + msg;
showToast(msg,'error');
return;
}
_agentBusy = true;
document.getElementById('agent-running-tag').style.display = 'inline';
document.getElementById('btn-agent-send').disabled = true;
listenStream(r.run_id);
});
}
function listenStream(runId){
if(_agentStream) _agentStream.close();
const es = new EventSource('/api/agent/stream?run_id=' + runId);
_agentStream = es;
let msgEl = null;
es.addEventListener('delta', ev=>{
if(!msgEl) msgEl = newAssistantMsg();
const d = JSON.parse(ev.data);
const textEl = msgEl.querySelector('.agent-text');
if(d.kind === 'reasoning'){
let r = msgEl.querySelector('.reasoning');
if(!r){
r = document.createElement('div');
r.className = 'reasoning';
msgEl.insertBefore(r, textEl);
}
r.textContent += d.text;
}else{
textEl.textContent += d.text;
}
scrollChat();
});
es.addEventListener('step', ev=>{
const d = JSON.parse(ev.data);
if(!msgEl) msgEl = newAssistantMsg();
const cards = msgEl.querySelector('.agent-toolcards');
const card = document.createElement('div');
card.className = 'agent-toolcard';
card.innerHTML = '<span class="dot"></span>'
+ '<code style="color:var(--teal)">' + esc(d.tool||'') + '</code>'
+ '<span class="text-muted">' + esc(d.args||'') + '</span>';
if(d.image){
const img = document.createElement('img');
img.src = 'data:image/jpeg;base64,' + d.image;
card.appendChild(img);
}
cards.appendChild(card);
scrollChat();
});
es.addEventListener('done', ev=>{
const d = JSON.parse(ev.data);
if(d.answer){
if(!msgEl) msgEl = newAssistantMsg();
msgEl.querySelector('.agent-text').textContent = d.answer;
}
endRun();
});
es.addEventListener('error', ev=>{
let msg = '连接中断';
try{
if(ev.data) msg = JSON.parse(ev.data).message || msg;
}catch(e){}
if(!msgEl) msgEl = newAssistantMsg();
msgEl.querySelector('.agent-text').textContent = '⚠ ' + msg;
endRun();
});
es.onerror = ()=>{
// 事件流正常结束(done 后服务器关流)会触发一次 error——done 已处理则忽略
if(_agentBusy) endRun();
};
}
function endRun(){
_agentBusy = false;
_agentStream && _agentStream.close();
_agentStream = null;
document.getElementById('agent-running-tag').style.display = 'none';
document.getElementById('btn-agent-send').disabled = false;
scrollChat();
}
+1 -1
View File
@@ -100,6 +100,7 @@ function showTab(name){
if(name==='tasks'){showSubTab('tasks',_activeSubs.tasks);loadTasks();loadCustomActions();}
if(name==='tools'){showSubTab('tools',_activeSubs.tools);loadToolsDevices();loadAdbDevices();loadTailscaleDevices();loadApks();}
if(name==='logs'){loadLogs();if(document.getElementById('log-auto').checked)_logTimer=setInterval(loadLogs,3000);}
if(name==='agent' && typeof initAgentChat==='function') initAgentChat();
if(name==='users')loadUsers();
}
@@ -114,7 +115,6 @@ function showSubTab(tabId, name){
tab.querySelectorAll('.sub-tab').forEach(b=>b.classList.toggle('active', b.dataset.sub===name));
tab.querySelectorAll('.sub-panel').forEach(p=>p.classList.toggle('active', p.id===tabId+'-sub-'+name));
if(name==='groups' && typeof loadGroups==='function') loadGroups();
if(name==='agent' && typeof loadAgentConfig==='function') loadAgentConfig();
if(name==='devpool' && typeof loadDevPool==='function'){
loadDevPool();
if(typeof loadDiscovery==='function'){
-83
View File
@@ -41,89 +41,6 @@ async function loadToolsDevices(force){
status.textContent = '共 '+_clipDevices.length+' 台设备';
}
// ================== AI 控制台(模型配置 + 指令执行 + 进度轮询) ==================
let _agentPoll = null;
function loadAgentConfig(){
apiGet('/api/agent/config').then(r=>{
if(!r||!r.ok)return;
const b=document.getElementById('agent-api-base');
if(!b)return; // 面板未渲染
b.value=r.api_base||'https://api.deepseek.com';
document.getElementById('agent-model').value=r.model||'';
document.getElementById('agent-default-serial').value=r.default_serial||'';
const hint=document.getElementById('agent-key-hint');
hint.textContent=r.api_key_masked?('已配置 '+r.api_key_masked):'未配置 Key';
});
}
function saveAgentConfig(){
const body={api_base:document.getElementById('agent-api-base').value.trim(),
model:document.getElementById('agent-model').value.trim(),
default_serial:document.getElementById('agent-default-serial').value.trim()};
const key=document.getElementById('agent-api-key').value.trim();
if(key)body.api_key=key;
apiPost('/api/agent/config',body).then(r=>{
if(r&&r.ok){showToast('配置已保存','success');loadAgentConfig();}
else showToast('保存失败: '+((r&&r.error)||''),'error');
});
}
function runAgent(){
const prompt=document.getElementById('agent-prompt').value.trim();
if(!prompt){showToast('请输入指令','error');return;}
apiPost('/api/agent/run',{prompt}).then(r=>{
if(r&&r.ok){
showToast('Agent 已启动','success');
document.getElementById('agent-status').textContent='运行中...';
document.getElementById('btn-agent-run').disabled=true;
document.getElementById('agent-answer-wrap').style.display='none';
startAgentPoll();
}else showToast('启动失败: '+((r&&r.error)||''),'error');
});
}
function startAgentPoll(){
if(_agentPoll)clearInterval(_agentPoll);
_agentPoll=setInterval(pollAgent,2000);
pollAgent();
}
function stopAgentPoll(){
if(_agentPoll){clearInterval(_agentPoll);_agentPoll=null;}
}
function pollAgent(){
apiGet('/api/agent/status').then(r=>{
if(!r||!r.ok)return;
const st=document.getElementById('agent-status');
if(r.state==='running'){
st.textContent='运行中... ('+(r.steps||[]).length+' 步)';
}else{
st.textContent=r.state==='done'?'完成':'失败';
document.getElementById('btn-agent-run').disabled=false;
stopAgentPoll();
}
// 步骤流(含截图缩略)
const steps=document.getElementById('agent-steps');
const html=(r.steps||[]).map(s=>{
const args=esc(s.args||'');
const img=s.image
?'<img src="data:image/jpeg;base64,'+s.image+'" style="max-width:120px;border-radius:6px;margin-top:4px;display:block">':'';
return '<div style="padding:6px 0;border-bottom:1px solid var(--ink-line)">'
+'<code style="color:var(--teal)">'+esc(s.tool||'')+'</code> '
+'<span class="text-muted">'+args+'</span>'+img+'</div>';
}).join('')||'';
steps.innerHTML=html;
// 最终回答
if(r.answer){
const wrap=document.getElementById('agent-answer-wrap');
wrap.style.display='block';
document.getElementById('agent-answer').textContent=r.answer;
}
if(r.error){
st.textContent='错误: '+r.error;
document.getElementById('btn-agent-run').disabled=false;
stopAgentPoll();
}
});
}
// ================== 设备自动发现(扫描 → 待连接池 → 确认) ==================
function loadDiscovery(){
apiGet('/api/devices/discovery').then(r=>{
+63 -31
View File
@@ -303,6 +303,27 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
.el-picker-item .ep-info .ep-attrs{color:var(--text-light);font-size:11px;margin-top:2px;word-break:break-all}
.el-picker-item .ep-info .ep-attrs span{margin-right:8px}
.el-picker-item .ep-sel{font-size:10px;color:#fff;background:var(--primary);padding:1px 7px;border-radius:99px;flex-shrink:0;font-weight:600}
/* ===== AI 控制台(聊天界面) ===== */
.agent-shell{display:flex;flex-direction:column;height:calc(100vh - 150px);min-height:480px;border:1px solid var(--card-line);border-radius:14px;overflow:hidden;background:#0d1117}
.agent-topbar{display:flex;align-items:center;justify-content:space-between;padding:10px 16px;border-bottom:1px solid var(--ink-line);background:#11151c}
.agent-title{font-size:14px;font-weight:700}
.agent-model-tag{margin-left:8px;font-size:11px;color:var(--teal);font-family:var(--mono);background:rgba(45,212,191,.08);padding:2px 8px;border-radius:10px}
.agent-running-tag{color:#f87171;font-size:12px;margin-right:8px}
.agent-top-actions{display:flex;align-items:center;gap:6px}
.agent-chat{flex:1;overflow-y:auto;padding:18px 22px;display:flex;flex-direction:column;gap:14px}
.agent-msg{max-width:78%;padding:10px 14px;border-radius:12px;font-size:13.5px;line-height:1.65;white-space:pre-wrap;word-break:break-word}
.agent-msg.user{align-self:flex-end;background:#155e63;color:#e6fffb;border-bottom-right-radius:3px}
.agent-msg.assistant{align-self:flex-start;background:#161c26;border:1px solid var(--card-line);border-bottom-left-radius:3px;color:var(--text)}
.agent-msg .reasoning{color:#8b95a7;font-size:12.5px;border-left:2px solid #2b3441;padding-left:8px;margin:6px 0}
.agent-toolcards{display:flex;flex-direction:column;gap:4px;margin-bottom:8px}
.agent-toolcard{display:flex;align-items:center;gap:8px;background:#0d1117;border:1px solid #232936;border-radius:8px;padding:5px 10px;font-size:12px;font-family:var(--mono);flex-wrap:wrap}
.agent-toolcard .dot{width:7px;height:7px;border-radius:50%;background:var(--teal);flex:none}
.agent-toolcard img{max-width:130px;border-radius:6px;margin-top:3px}
.agent-inputbar{display:flex;gap:10px;padding:12px 14px;border-top:1px solid var(--ink-line);background:#11151c}
.agent-inputbar textarea{flex:1;resize:none;background:#0d1117;border:1px solid var(--card-line);border-radius:10px;padding:9px 12px;color:var(--text);font-size:13.5px;outline:none;line-height:1.5}
.agent-inputbar textarea:focus{border-color:var(--teal)}
.agent-empty{color:#4b5563;font-size:13px;text-align:center;padding:40px 0}
</style>
</head>
<body>
@@ -316,6 +337,7 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
<button class="tab" data-tab="logs" onclick="showTab('logs')" data-perm="logs">日志</button>
<button class="tab" data-tab="users" onclick="showTab('users')" data-perm="admin">用户</button>
<button class="tab" data-tab="tools" onclick="showTab('tools')" data-perm="admin">工具</button>
<button class="tab" data-tab="agent" onclick="showTab('agent')" data-perm="admin">AI 控制台</button>
</div>
<div class="user-info">
<span id="me-username">...</span>
@@ -422,6 +444,45 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
</div>
<!-- ========== Tab 3: 分组 ========== -->
<!-- ========== Tab: AI 控制台(DeepSeek 风格聊天 + 实时 MCP 步骤) ========== -->
<div id="tab-agent" class="tab-panel">
<div class="agent-shell">
<div class="agent-topbar">
<div class="agent-title">🤖 AI 控制台<span id="agent-model-tag" class="agent-model-tag"></span></div>
<div class="agent-top-actions">
<span id="agent-running-tag" class="agent-running-tag" style="display:none">● 运行中</span>
<button class="btn btn-xs" onclick="clearAgentChat()">清空对话</button>
<button class="btn btn-xs" onclick="openAgentConfig()">⚙ 配置</button>
</div>
</div>
<div id="agent-chat" class="agent-chat"></div>
<div class="agent-inputbar">
<textarea id="agent-input" rows="2"
placeholder="给 AI 下达指令,例如:打开抖音搜索奚学东,把第一个视频标题告诉我…(Enter 发送,Shift+Enter 换行)"></textarea>
<button class="btn btn-primary" id="btn-agent-send" onclick="sendAgentMsg()">发送</button>
</div>
</div>
</div>
<!-- ========== 配置模态框 ========== -->
<div id="agent-cfg-overlay" style="display:none;position:fixed;inset:0;background:rgba(0,0,0,.6);z-index:1300;align-items:center;justify-content:center">
<div style="background:#11151c;border:1px solid var(--card-line);border-radius:14px;padding:22px;width:460px;max-width:92vw">
<div style="font-size:15px;font-weight:700;margin-bottom:14px">模型配置(OpenAI 兼容 API)</div>
<div class="form-group"><label>API Base</label>
<input type="text" id="acfg-base" class="form-control" placeholder="https://api.deepseek.com" style="font-family:monospace"></div>
<div class="form-group"><label>模型名</label>
<input type="text" id="acfg-model" class="form-control" placeholder="deepseek-v4-flash-vision-exp"></div>
<div class="form-group"><label>API Key <span class="text-muted" id="acfg-key-hint"></span></label>
<input type="password" id="acfg-key" class="form-control" placeholder="留空则不修改"></div>
<div class="form-group"><label>默认设备 serial(可选)</label>
<input type="text" id="acfg-serial" class="form-control" placeholder="192.168.20.66:5555" style="font-family:monospace"></div>
<div style="display:flex;justify-content:flex-end;gap:8px;margin-top:6px">
<button class="btn" onclick="closeAgentConfig()">取消</button>
<button class="btn btn-primary" onclick="saveAgentConfig()">保存</button>
</div>
</div>
</div>
<!-- ========== Tab 4: 日志 ========== -->
<div id="tab-logs" class="tab-panel">
<div class="page-title">日志查看</div>
@@ -480,7 +541,6 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
<button class="sub-tab" data-sub="devapps" onclick="showSubTab('tools','devapps')">设备已装应用</button>
<button class="sub-tab" data-sub="devpool" onclick="showSubTab('tools','devpool')">设备池管理</button>
<button class="sub-tab" data-sub="groups" onclick="showSubTab('tools','groups')">设备分组</button>
<button class="sub-tab" data-sub="agent" onclick="showSubTab('tools','agent')">AI 控制台</button>
</div>
<!-- 子分栏 1:剪贴板注入 -->
@@ -703,36 +763,7 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
<div class="help">管理设备分组,用于任务按分组执行。分组内的设备需先在<b>设备池管理</b>中确认(不在池内的设备不参与调度)。</div>
</div>
<!-- 子分栏:AI 控制台(第三方模型经 Agent 控制手机) -->
<div class="sub-panel" id="tools-sub-agent">
<details style="margin-bottom:10px">
<summary style="cursor:pointer;color:var(--muted)">⚙ 模型配置(OpenAI 兼容 API)</summary>
<div style="padding:10px 0 4px;display:flex;flex-wrap:wrap;gap:8px;align-items:center">
<input type="text" id="agent-api-base" class="form-control" style="width:260px;font-family:monospace"
placeholder="API Base,如 https://api.deepseek.com">
<input type="text" id="agent-model" class="form-control" style="width:220px"
placeholder="模型名,如 deepseek-v4-flash-vision-exp">
<input type="password" id="agent-api-key" class="form-control" style="width:260px"
placeholder="API Key(留空不改)">
<input type="text" id="agent-default-serial" class="form-control" style="width:220px;font-family:monospace"
placeholder="默认设备 serial(可选)">
<button class="btn btn-primary" onclick="saveAgentConfig()">保存配置</button>
<span id="agent-key-hint" class="text-muted"></span>
</div>
</details>
<div class="toolbar" style="flex-wrap:wrap">
<textarea id="agent-prompt" class="form-control" rows="2" style="flex:1;min-width:300px"
placeholder="指令:如「打开抖音搜索奚学东,截个图描述结果」"></textarea>
<button class="btn btn-primary" id="btn-agent-run" onclick="runAgent()">▶ 执行</button>
<span id="agent-status" class="text-muted"></span>
</div>
<div id="agent-steps" style="margin-top:10px"></div>
<div id="agent-answer-wrap" style="display:none;margin-top:12px">
<div class="section-title">回答</div>
<div id="agent-answer" style="white-space:pre-wrap;line-height:1.7;background:#0d1117;border:1px solid var(--card-line);border-radius:10px;padding:12px"></div>
</div>
<div class="help">AI 通过截图观察屏幕并执行操作(元素定位优先,坐标兜底)。模型与 Key 配置保存在平台(管理员可见),请勿填他人密钥。</div>
</div>
</div>
@@ -798,5 +829,6 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
<script src="/static/admin/tools.js"></script>
<script src="/static/admin/apps.js"></script>
<script src="/static/admin/admin.js"></script>
<script src="/static/admin/agent.js"></script>
</body>
</html>
+70 -29
View File
@@ -1,17 +1,27 @@
"""AI 控制台 API:Agent 配置(模型/Key 前端可配)+ 运行 + 状态轮询。
"""AI 控制台 API:模型/Key 前端配置 + Agent 流式执行(SSE 事件流)。
Agent 在平台进程内跑(后台线程),连本机 MCP Server(8033)执行工具。
单实例:同一时间只允许一个 Agent 运行,避免多路操作设备冲突。
流程:
POST /api/agent/run {prompt} → 启动 Agent 线程,返回 run_id
GET /api/agent/stream?run_id= → SSE 事件流(EventSource 订阅):
event: delta {text, kind: content|reasoning} 流式文本增量
event: step {tool, args, image?} 工具调用完成(MCP 步骤)
event: done {answer} 完成
event: error {message} 失败
GET/POST /api/agent/config → 配置读写(key 打码回显)
单实例:同时只允许一个 Agent 运行。
"""
import asyncio
import base64
import io
import json
import os
import queue
import sys
import threading
import uuid
from flask import Blueprint, jsonify, request
from flask import Blueprint, Response, jsonify, request
from core.logger import get_logger
from core.models import db
@@ -26,9 +36,10 @@ _CFG_KEYS = {"api_base": "agent_api_base",
"api_key": "agent_api_key",
"default_serial": "agent_default_serial"}
# ---------- 运行状态(单实例) ----------
# ---------- 运行状态(单实例 + 事件队列) ----------
_run = {"id": None, "state": "idle", "prompt": "", "serial": "",
"steps": [], "answer": "", "error": ""}
"answer": "", "error": ""}
_queues = {} # run_id -> queue.Queue(SSE 消费者读取)
_lock = threading.Lock()
@@ -47,6 +58,7 @@ def _read_cfg():
return {k: _meta_get(v) for k, v in _CFG_KEYS.items()}
# ================== 配置 ==================
@bp.route("/api/agent/config", methods=["GET"])
@admin_required
def agent_config_get():
@@ -70,10 +82,11 @@ def agent_config_save():
return jsonify({"ok": True, "msg": "已保存"})
# ================== 运行 ==================
@bp.route("/api/agent/run", methods=["POST"])
@admin_required
def agent_run():
"""启动 Agent 执行指令:{prompt, serial?}。运行中返回 409。"""
"""启动 Agent:{prompt, serial?}。运行中返回 409。"""
data = request.json or {}
prompt = (data.get("prompt") or "").strip()
if not prompt:
@@ -81,29 +94,53 @@ def agent_run():
cfg = _read_cfg()
if not cfg.get("api_key"):
return jsonify({"ok": False, "error": "请先在配置区填写 API Key"}), 400
if not cfg.get("model"):
return jsonify({"ok": False, "error": "请先填写模型名"}), 400
with _lock:
if _run["state"] == "running":
return jsonify({"ok": False, "error": "已有 Agent 运行中,请等待完成"}), 409
run_id = uuid.uuid4().hex[:8]
_run.update(id=run_id, state="running", prompt=prompt,
serial=(data.get("serial") or "").strip(),
steps=[], answer="", error="")
_log.info(f"Agent 启动: {prompt[:60]} serial={_run['serial']}")
answer="", error="")
_queues[run_id] = queue.Queue()
_log.info(f"Agent 启动: {prompt[:60]}")
threading.Thread(target=_agent_thread, args=(run_id, prompt, cfg),
daemon=True).start()
return jsonify({"ok": True, "run_id": run_id})
@bp.route("/api/agent/status")
@bp.route("/api/agent/stream")
@admin_required
def agent_status():
"""Agent 运行状态(前端轮询):{state, prompt, serial, steps, answer, error}。"""
def agent_stream():
"""SSE 事件流(EventSource):delta/step/done/error。"""
run_id = request.args.get("run_id", "")
with _lock:
return jsonify({"ok": True, **_run})
if run_id != _run["id"]:
return jsonify({"ok": False, "error": "run_id 不存在"}), 404
q = _queues.get(run_id)
def gen():
while True:
try:
evt = q.get(timeout=15)
except queue.Empty:
yield ": keepalive\n\n" # 心跳防超时
continue
if evt is None:
break
kind, payload = evt
yield f"event: {kind}\ndata: {json.dumps(payload, ensure_ascii=False)}\n\n"
if kind in ("done", "error"):
break
return Response(gen(), mimetype="text/event-stream",
headers={"Cache-Control": "no-cache",
"X-Accel-Buffering": "no"})
def _shrink_image(b64, width=220, quality=50):
"""截图降采样(web 展示用,避免大图撑爆轮询响应)。失败原样返回。"""
"""截图降采样(SSE step 事件用,控制传输体积)。失败原样返回。"""
try:
from PIL import Image
img = Image.open(io.BytesIO(base64.b64decode(b64)))
@@ -117,42 +154,46 @@ def _shrink_image(b64, width=220, quality=50):
def _agent_thread(run_id, prompt, cfg):
"""后台线程:跑 Agent,回调记录步骤。"""
"""后台线程:Agent 流式执行,事件推入队列供 SSE 消费。"""
q = _queues.get(run_id)
try:
# 平台进程 cwd=/app(含 mcp_agent 包),进程内 import
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from mcp_agent.agent import Agent
def on_step(step):
with _lock:
if _run["id"] != run_id:
return
rec = {"tool": step.get("tool"), "args": str(step.get("args"))[:120]}
if step.get("image_b64"):
rec["image"] = _shrink_image(step["image_b64"])
# 保留最近 6 步截图,防止响应过大
_run["steps"] = _run["steps"][-5:]
_run["steps"].append(rec)
def on_delta(text, kind):
q.put(("delta", {"text": text, "kind": kind}))
def on_tool(step):
rec = {"tool": step.get("tool"),
"args": str(step.get("args"))[:200]}
if step.get("image"):
rec["image"] = _shrink_image(step["image"])
q.put(("step", rec))
agent = Agent()
# 用 web 配置覆盖 Agent 默认(api key 等由前端配置)
agent.s.api_base = cfg.get("api_base") or agent.s.api_base
agent.s.model = cfg.get("model") or agent.s.model
agent.s.api_key = cfg.get("api_key") or agent.s.api_key
agent.s.default_serial = cfg.get("default_serial") or agent.s.default_serial
agent.on_step = on_step
async def _execute():
await agent._load_tools()
return await agent.run(prompt, cfg.get("default_serial") or "")
return await agent.run_stream(prompt, cfg.get("default_serial") or "",
on_delta=on_delta, on_tool=on_tool)
# 整体超时保护:模型/工具卡死时结束运行,避免单实例被永久占用
# 整体超时保护:卡死时结束,释放单实例
answer = asyncio.run(asyncio.wait_for(_execute(), timeout=600))
with _lock:
_run["state"] = "done"
_run["answer"] = answer
q.put(("done", {"answer": answer}))
except Exception as e:
_log.warning(f"Agent 运行异常: {e}")
with _lock:
_run["state"] = "error"
_run["error"] = f"{type(e).__name__}: {str(e)[:200]}"
q.put(("error", {"message": str(e)[:200]}))
finally:
q.put(None) # 关闭 SSE
_queues.pop(run_id, None)