feat: AI 控制台升级为顶级 Tab——DeepSeek 风格聊天界面(气泡+流式渲染+思考折叠),实时 MCP 步骤卡片(工具/参数/截图缩略),SSE 流式输出(delta/step/done/error 事件),配置模态(模型/Key/设备前端可配);工具页旧子栏移除
This commit is contained in:
+110
-54
@@ -1,11 +1,13 @@
|
||||
"""Agent 编排层:OpenAI 兼容模型(DeepSeek 等)经 MCP 工具控制手机。
|
||||
|
||||
工作流:
|
||||
1. 启动时从 MCP Server 拉工具列表 → 转 OpenAI function schema
|
||||
2. run(prompt):循环 chat/completions
|
||||
- 模型返回 tool_calls → 依次执行(经 MCP)→ 结果回喂
|
||||
- de_screenshot 的返回图像转为 image_url 追加为下一轮 user 消息(多模态看图)
|
||||
- 无 tool_calls → 返回最终文本
|
||||
支持两种运行模式:
|
||||
- run_stream():流式(SSE 逐 token + 工具调用实时回调)——Web AI 控制台用
|
||||
- run():非流式收集结果——CLI 用(内部调 run_stream)
|
||||
|
||||
流式细节(OpenAI 兼容):
|
||||
- content/reasoning_content 增量逐 chunk 回调(kind 区分)
|
||||
- tool_calls 分片累积(arguments 按 index 拼接),流结束后统一执行
|
||||
- 截图(de_screenshot)图像转 image_url 追加下一轮,同时 on_tool 回调带缩略
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
@@ -26,9 +28,10 @@ SYSTEM_PROMPT = """你是手机自动化控制助手。你通过工具实时操
|
||||
1. 先 de_list_devices 确定目标设备(在线才可操作)
|
||||
2. 观察屏幕:先 de_screenshot 获取截图(图像会随后给你),基于截图理解当前界面
|
||||
3. 操作:de_tap/de_swipe 的坐标必须与最近一次 de_screenshot 图像一致(直接看图给坐标,服务器自动换算)
|
||||
4. 每次关键操作后再次 de_screenshot 验证结果,直到完成用户目标
|
||||
5. 完成或失败时用中文总结:做了什么、当前状态、需要用户注意的事项
|
||||
6. 设备不可用/操作失败时如实报告错误,不要臆测成功
|
||||
4. 元素操作优先:能用 de_ui_tree/de_tap_element(text/id/desc 定位)就不用裸坐标
|
||||
5. 每次关键操作后再次 de_screenshot 验证结果,直到完成用户目标
|
||||
6. 完成或失败时用中文总结:做了什么、当前状态、需要用户注意的事项
|
||||
7. 设备不可用/操作失败时如实报告错误,不要臆测成功
|
||||
|
||||
可用工具清单将由系统提供。"""
|
||||
|
||||
@@ -37,9 +40,13 @@ class Agent:
|
||||
def __init__(self, settings: AgentSettings = None):
|
||||
self.s = settings or S
|
||||
self.tools_schema = [] # OpenAI function schema
|
||||
self._tool_exec = {} # name -> callable
|
||||
self.messages = []
|
||||
self.on_step = None # 可选回调 fn(step_dict),web 展示进度用
|
||||
# 回调(Web 展示用,均可选):
|
||||
# on_delta(text, kind) kind: content | reasoning —— 流式文本增量
|
||||
# on_tool(step) step: {tool, args, result, image} —— 工具调用完成
|
||||
self.on_delta = None
|
||||
self.on_tool = None
|
||||
self._mcp = None
|
||||
|
||||
# ---------- MCP 工具桥 ----------
|
||||
async def _load_tools(self):
|
||||
@@ -52,37 +59,49 @@ class Agent:
|
||||
for t in tools:
|
||||
# MCP SDK v2 改名 input_schema,兼容新旧字段
|
||||
schema = getattr(t, "input_schema", None) or getattr(t, "inputSchema", {})
|
||||
# fastmcp Tool 属性兼容:name/description/inputSchema
|
||||
name = getattr(t, "name", "")
|
||||
desc = getattr(t, "description", "") or ""
|
||||
self.tools_schema.append({
|
||||
"type": "function",
|
||||
"function": {"name": name, "description": desc,
|
||||
"parameters": schema}})
|
||||
self._tool_exec[name] = t
|
||||
_log.info("MCP 工具已加载: %s", [s["function"]["name"] for s in self.tools_schema])
|
||||
|
||||
async def close(self):
|
||||
if getattr(self, "_mcp", None):
|
||||
await self._mcp.__aexit__(None, None, None)
|
||||
if self._mcp:
|
||||
try:
|
||||
await self._mcp.__aexit__(None, None, None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ---------- 模型调用 ----------
|
||||
async def _chat(self):
|
||||
"""调用 OpenAI 兼容 chat/completions,返回完整 response JSON。"""
|
||||
# ---------- 模型调用(流式) ----------
|
||||
async def _chat_stream(self):
|
||||
"""流式 chat/completions:逐 chunk 产出 JSON(async generator)。"""
|
||||
body = {
|
||||
"model": self.s.model,
|
||||
"messages": self.messages,
|
||||
"tools": self.tools_schema if self.tools_schema else None,
|
||||
"max_tokens": 4096,
|
||||
"stream": True,
|
||||
}
|
||||
headers = {"Authorization": f"Bearer {self.s.api_key}",
|
||||
"Content-Type": "application/json"}
|
||||
url = f"{self.s.api_base.rstrip('/')}/chat/completions"
|
||||
async with httpx.AsyncClient(timeout=self.s.request_timeout) as client:
|
||||
r = await client.post(f"{self.s.api_base.rstrip('/')}/chat/completions",
|
||||
json=body, headers=headers)
|
||||
if r.status_code != 200:
|
||||
raise RuntimeError(f"模型 API HTTP {r.status_code}: {r.text[:300]}")
|
||||
return r.json()
|
||||
async with client.stream("POST", url, json=body, headers=headers) as r:
|
||||
if r.status_code != 200:
|
||||
text = (await r.aread()).decode(errors="replace")
|
||||
raise RuntimeError(f"模型 API HTTP {r.status_code}: {text[:300]}")
|
||||
async for line in r.aiter_lines():
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
data = line[5:].strip()
|
||||
if data == "[DONE]":
|
||||
break
|
||||
try:
|
||||
yield json.loads(data)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
# ---------- 工具执行 ----------
|
||||
async def _execute_tool(self, name, arguments):
|
||||
@@ -94,28 +113,33 @@ class Agent:
|
||||
data = getattr(result, "data", result)
|
||||
except Exception as e:
|
||||
return {"ok": False, "error": f"工具执行失败: {e}"}, None
|
||||
# de_screenshot:图像分离(作为 image_url 追加给模型看)
|
||||
# de_screenshot:图像分离(作为 image_url 追加给模型看 + on_tool 缩略展示)
|
||||
image_b64 = None
|
||||
text_result = data
|
||||
if name == "de_screenshot" and isinstance(data, dict) and data.get("ok"):
|
||||
img = (data.get("data") or {}).get("image") or {}
|
||||
if img.get("data"):
|
||||
text_result = {k: v for k, v in (data.get("data") or {}).items()
|
||||
if k != "image"}
|
||||
image_b64 = img["data"]
|
||||
if self.on_step:
|
||||
if self.on_tool:
|
||||
try:
|
||||
self.on_step({"tool": name, "args": args,
|
||||
"result": text_result if image_b64 else data,
|
||||
"image_b64": image_b64})
|
||||
self.on_tool({"tool": name, "args": args,
|
||||
"result": text_result, "image": image_b64})
|
||||
except Exception:
|
||||
pass
|
||||
if image_b64:
|
||||
return text_result, image_b64
|
||||
return data, None
|
||||
return text_result, image_b64
|
||||
|
||||
# ---------- 主循环 ----------
|
||||
async def run(self, prompt: str, serial: str = "") -> str:
|
||||
"""执行一轮指令,返回最终回答文本。"""
|
||||
# ---------- 主循环(流式) ----------
|
||||
async def run_stream(self, prompt: str, serial: str = "",
|
||||
on_delta=None, on_tool=None):
|
||||
"""流式执行一轮指令,返回最终完整文本。
|
||||
|
||||
on_delta(text, kind):content/reasoning 文本增量(实时推给前端)
|
||||
on_tool(step):工具调用完成(实时显示 MCP 步骤)
|
||||
"""
|
||||
self.on_delta = on_delta
|
||||
self.on_tool = on_tool
|
||||
target = serial or self.s.default_serial
|
||||
sys_txt = SYSTEM_PROMPT
|
||||
if target:
|
||||
@@ -123,28 +147,55 @@ class Agent:
|
||||
self.messages = [{"role": "system", "content": sys_txt},
|
||||
{"role": "user", "content": prompt}]
|
||||
|
||||
for step in range(self.s.max_steps):
|
||||
resp = await self._chat()
|
||||
choice = (resp.get("choices") or [{}])[0]
|
||||
msg = choice.get("message") or {}
|
||||
|
||||
# 1) 工具调用
|
||||
tool_calls = msg.get("tool_calls")
|
||||
if tool_calls:
|
||||
self.messages.append({
|
||||
"role": "assistant",
|
||||
"content": msg.get("content") or "",
|
||||
"tool_calls": tool_calls})
|
||||
for tc in tool_calls:
|
||||
for _step in range(self.s.max_steps):
|
||||
content_parts = []
|
||||
tool_acc = {} # index -> {id, name, args}
|
||||
has_tool = False
|
||||
async for chunk in self._chat_stream():
|
||||
choice = (chunk.get("choices") or [{}])[0]
|
||||
delta = choice.get("delta") or {}
|
||||
text = delta.get("content")
|
||||
if text:
|
||||
content_parts.append(text)
|
||||
if on_delta:
|
||||
on_delta(text, "content")
|
||||
rtext = delta.get("reasoning_content")
|
||||
if rtext:
|
||||
if on_delta:
|
||||
on_delta(rtext, "reasoning")
|
||||
for tc in delta.get("tool_calls") or []:
|
||||
has_tool = True
|
||||
idx = tc.get("index", 0)
|
||||
acc = tool_acc.setdefault(idx, {"id": "", "name": "", "args": ""})
|
||||
if tc.get("id"):
|
||||
acc["id"] = tc["id"]
|
||||
fn = tc.get("function") or {}
|
||||
name = fn.get("name", "")
|
||||
if fn.get("name"):
|
||||
acc["name"] += fn["name"]
|
||||
if fn.get("arguments"):
|
||||
acc["args"] += fn["arguments"]
|
||||
|
||||
full_content = "".join(content_parts)
|
||||
|
||||
if has_tool:
|
||||
# 组装 assistant 消息(含 tool_calls)并执行工具
|
||||
tcs = []
|
||||
for idx in sorted(tool_acc):
|
||||
acc = tool_acc[idx]
|
||||
tcs.append({"id": acc["id"] or f"call_{idx}",
|
||||
"type": "function",
|
||||
"function": {"name": acc["name"],
|
||||
"arguments": acc["args"]}})
|
||||
self.messages.append({"role": "assistant",
|
||||
"content": full_content,
|
||||
"tool_calls": tcs})
|
||||
for tc in tcs:
|
||||
fn = tc["function"]
|
||||
text_result, image_b64 = await self._execute_tool(
|
||||
name, fn.get("arguments", "{}"))
|
||||
fn["name"], fn["arguments"])
|
||||
self.messages.append({
|
||||
"role": "tool",
|
||||
"tool_call_id": tc.get("id", ""),
|
||||
"role": "tool", "tool_call_id": tc["id"],
|
||||
"content": json.dumps(text_result, ensure_ascii=False)[:4000]})
|
||||
# 截图图像:作为下一轮 user 图像内容(OpenAI 协议 tool 结果只能文本)
|
||||
if image_b64:
|
||||
self.messages.append({
|
||||
"role": "user",
|
||||
@@ -155,7 +206,12 @@ class Agent:
|
||||
f"data:image/jpeg;base64,{image_b64}"}}]})
|
||||
continue
|
||||
|
||||
# 2) 最终回答
|
||||
return msg.get("content") or "(模型无输出)"
|
||||
# 无工具调用:本轮即最终回答
|
||||
return full_content
|
||||
|
||||
return "(达到最大步骤数未完成,请检查操作是否卡在循环)"
|
||||
|
||||
# ---------- 非流式(CLI) ----------
|
||||
async def run(self, prompt: str, serial: str = "") -> str:
|
||||
"""非流式执行,返回最终文本(CLI 用,内部走流式收集)。"""
|
||||
return await self.run_stream(prompt, serial)
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
// AI 控制台(顶级 Tab):DeepSeek 风格聊天 + 流式输出 + 实时 MCP 步骤
|
||||
let _agentBusy = false;
|
||||
let _agentStream = null; // EventSource
|
||||
let _agentCfgLoaded = false;
|
||||
|
||||
// ================== 初始化 ==================
|
||||
function initAgentChat(){
|
||||
if(_agentCfgLoaded)return;
|
||||
_agentCfgLoaded = true;
|
||||
loadAgentConfig();
|
||||
const chat = document.getElementById('agent-chat');
|
||||
if(chat && !chat.children.length){
|
||||
chat.innerHTML = '<div class="agent-empty">👋 给 AI 下达指令,它将通过截图观察手机并执行操作。<br>'
|
||||
+ '例如:「打开抖音搜索奚学东,告诉我第一个视频的标题」<br>'
|
||||
+ '点击右上角 ⚙ 配置模型与 API Key。</div>';
|
||||
}
|
||||
// 输入框快捷键
|
||||
const inp = document.getElementById('agent-input');
|
||||
inp.addEventListener('keydown', ev=>{
|
||||
if(ev.key==='Enter' && !ev.shiftKey){
|
||||
ev.preventDefault();
|
||||
if(!_agentBusy) sendAgentMsg();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ================== 配置 ==================
|
||||
function loadAgentConfig(){
|
||||
apiGet('/api/agent/config').then(r=>{
|
||||
if(!r||!r.ok)return;
|
||||
const tag = document.getElementById('agent-model-tag');
|
||||
if(tag && r.model) tag.textContent = r.model;
|
||||
});
|
||||
}
|
||||
function openAgentConfig(){
|
||||
apiGet('/api/agent/config').then(r=>{
|
||||
if(!r||!r.ok)return;
|
||||
document.getElementById('acfg-base').value = r.api_base || 'https://api.deepseek.com';
|
||||
document.getElementById('acfg-model').value = r.model || '';
|
||||
document.getElementById('acfg-serial').value = r.default_serial || '';
|
||||
document.getElementById('acfg-key-hint').textContent =
|
||||
r.api_key_masked ? ('已配置 ' + r.api_key_masked) : '未配置';
|
||||
document.getElementById('acfg-key').value = '';
|
||||
document.getElementById('agent-cfg-overlay').style.display = 'flex';
|
||||
});
|
||||
}
|
||||
function closeAgentConfig(){
|
||||
document.getElementById('agent-cfg-overlay').style.display = 'none';
|
||||
}
|
||||
function saveAgentConfig(){
|
||||
const body = {
|
||||
api_base: document.getElementById('acfg-base').value.trim(),
|
||||
model: document.getElementById('acfg-model').value.trim(),
|
||||
default_serial: document.getElementById('acfg-serial').value.trim(),
|
||||
};
|
||||
const key = document.getElementById('acfg-key').value.trim();
|
||||
if(key) body.api_key = key;
|
||||
apiPost('/api/agent/config', body).then(r=>{
|
||||
if(r&&r.ok){
|
||||
showToast('配置已保存','success');
|
||||
closeAgentConfig();
|
||||
loadAgentConfig();
|
||||
}else showToast('保存失败: ' + ((r&&r.error)||''),'error');
|
||||
});
|
||||
}
|
||||
|
||||
// ================== 聊天渲染 ==================
|
||||
function addUserMsg(text){
|
||||
const chat = document.getElementById('agent-chat');
|
||||
clearEmpty();
|
||||
const div = document.createElement('div');
|
||||
div.className = 'agent-msg user';
|
||||
div.textContent = text;
|
||||
chat.appendChild(div);
|
||||
scrollChat();
|
||||
}
|
||||
function newAssistantMsg(){
|
||||
const chat = document.getElementById('agent-chat');
|
||||
clearEmpty();
|
||||
const div = document.createElement('div');
|
||||
div.className = 'agent-msg assistant';
|
||||
div.innerHTML = '<div class="agent-toolcards"></div><div class="agent-text"></div>';
|
||||
chat.appendChild(div);
|
||||
scrollChat();
|
||||
return div;
|
||||
}
|
||||
function clearEmpty(){
|
||||
const empty = document.querySelector('#agent-chat .agent-empty');
|
||||
if(empty) empty.remove();
|
||||
}
|
||||
function scrollChat(){
|
||||
const chat = document.getElementById('agent-chat');
|
||||
chat.scrollTop = chat.scrollHeight;
|
||||
}
|
||||
function clearAgentChat(){
|
||||
document.getElementById('agent-chat').innerHTML =
|
||||
'<div class="agent-empty">对话已清空。给 AI 下达新指令…</div>';
|
||||
}
|
||||
|
||||
// ================== 发送与流式接收 ==================
|
||||
function sendAgentMsg(){
|
||||
if(_agentBusy){showToast('上一轮还在运行','error');return;}
|
||||
const inp = document.getElementById('agent-input');
|
||||
const text = inp.value.trim();
|
||||
if(!text){showToast('请输入指令','error');return;}
|
||||
inp.value = '';
|
||||
addUserMsg(text);
|
||||
|
||||
apiPost('/api/agent/run', {prompt: text}).then(r=>{
|
||||
if(!r||!r.ok){
|
||||
const msg = r ? (r.error||'启动失败') : '请求失败';
|
||||
const div = newAssistantMsg();
|
||||
div.querySelector('.agent-text').textContent = '⚠ ' + msg;
|
||||
showToast(msg,'error');
|
||||
return;
|
||||
}
|
||||
_agentBusy = true;
|
||||
document.getElementById('agent-running-tag').style.display = 'inline';
|
||||
document.getElementById('btn-agent-send').disabled = true;
|
||||
listenStream(r.run_id);
|
||||
});
|
||||
}
|
||||
|
||||
function listenStream(runId){
|
||||
if(_agentStream) _agentStream.close();
|
||||
const es = new EventSource('/api/agent/stream?run_id=' + runId);
|
||||
_agentStream = es;
|
||||
let msgEl = null;
|
||||
|
||||
es.addEventListener('delta', ev=>{
|
||||
if(!msgEl) msgEl = newAssistantMsg();
|
||||
const d = JSON.parse(ev.data);
|
||||
const textEl = msgEl.querySelector('.agent-text');
|
||||
if(d.kind === 'reasoning'){
|
||||
let r = msgEl.querySelector('.reasoning');
|
||||
if(!r){
|
||||
r = document.createElement('div');
|
||||
r.className = 'reasoning';
|
||||
msgEl.insertBefore(r, textEl);
|
||||
}
|
||||
r.textContent += d.text;
|
||||
}else{
|
||||
textEl.textContent += d.text;
|
||||
}
|
||||
scrollChat();
|
||||
});
|
||||
|
||||
es.addEventListener('step', ev=>{
|
||||
const d = JSON.parse(ev.data);
|
||||
if(!msgEl) msgEl = newAssistantMsg();
|
||||
const cards = msgEl.querySelector('.agent-toolcards');
|
||||
const card = document.createElement('div');
|
||||
card.className = 'agent-toolcard';
|
||||
card.innerHTML = '<span class="dot"></span>'
|
||||
+ '<code style="color:var(--teal)">' + esc(d.tool||'') + '</code>'
|
||||
+ '<span class="text-muted">' + esc(d.args||'') + '</span>';
|
||||
if(d.image){
|
||||
const img = document.createElement('img');
|
||||
img.src = 'data:image/jpeg;base64,' + d.image;
|
||||
card.appendChild(img);
|
||||
}
|
||||
cards.appendChild(card);
|
||||
scrollChat();
|
||||
});
|
||||
|
||||
es.addEventListener('done', ev=>{
|
||||
const d = JSON.parse(ev.data);
|
||||
if(d.answer){
|
||||
if(!msgEl) msgEl = newAssistantMsg();
|
||||
msgEl.querySelector('.agent-text').textContent = d.answer;
|
||||
}
|
||||
endRun();
|
||||
});
|
||||
|
||||
es.addEventListener('error', ev=>{
|
||||
let msg = '连接中断';
|
||||
try{
|
||||
if(ev.data) msg = JSON.parse(ev.data).message || msg;
|
||||
}catch(e){}
|
||||
if(!msgEl) msgEl = newAssistantMsg();
|
||||
msgEl.querySelector('.agent-text').textContent = '⚠ ' + msg;
|
||||
endRun();
|
||||
});
|
||||
|
||||
es.onerror = ()=>{
|
||||
// 事件流正常结束(done 后服务器关流)会触发一次 error——done 已处理则忽略
|
||||
if(_agentBusy) endRun();
|
||||
};
|
||||
}
|
||||
|
||||
function endRun(){
|
||||
_agentBusy = false;
|
||||
_agentStream && _agentStream.close();
|
||||
_agentStream = null;
|
||||
document.getElementById('agent-running-tag').style.display = 'none';
|
||||
document.getElementById('btn-agent-send').disabled = false;
|
||||
scrollChat();
|
||||
}
|
||||
@@ -100,6 +100,7 @@ function showTab(name){
|
||||
if(name==='tasks'){showSubTab('tasks',_activeSubs.tasks);loadTasks();loadCustomActions();}
|
||||
if(name==='tools'){showSubTab('tools',_activeSubs.tools);loadToolsDevices();loadAdbDevices();loadTailscaleDevices();loadApks();}
|
||||
if(name==='logs'){loadLogs();if(document.getElementById('log-auto').checked)_logTimer=setInterval(loadLogs,3000);}
|
||||
if(name==='agent' && typeof initAgentChat==='function') initAgentChat();
|
||||
if(name==='users')loadUsers();
|
||||
}
|
||||
|
||||
@@ -114,7 +115,6 @@ function showSubTab(tabId, name){
|
||||
tab.querySelectorAll('.sub-tab').forEach(b=>b.classList.toggle('active', b.dataset.sub===name));
|
||||
tab.querySelectorAll('.sub-panel').forEach(p=>p.classList.toggle('active', p.id===tabId+'-sub-'+name));
|
||||
if(name==='groups' && typeof loadGroups==='function') loadGroups();
|
||||
if(name==='agent' && typeof loadAgentConfig==='function') loadAgentConfig();
|
||||
if(name==='devpool' && typeof loadDevPool==='function'){
|
||||
loadDevPool();
|
||||
if(typeof loadDiscovery==='function'){
|
||||
|
||||
@@ -41,89 +41,6 @@ async function loadToolsDevices(force){
|
||||
status.textContent = '共 '+_clipDevices.length+' 台设备';
|
||||
}
|
||||
|
||||
// ================== AI 控制台(模型配置 + 指令执行 + 进度轮询) ==================
|
||||
let _agentPoll = null;
|
||||
|
||||
function loadAgentConfig(){
|
||||
apiGet('/api/agent/config').then(r=>{
|
||||
if(!r||!r.ok)return;
|
||||
const b=document.getElementById('agent-api-base');
|
||||
if(!b)return; // 面板未渲染
|
||||
b.value=r.api_base||'https://api.deepseek.com';
|
||||
document.getElementById('agent-model').value=r.model||'';
|
||||
document.getElementById('agent-default-serial').value=r.default_serial||'';
|
||||
const hint=document.getElementById('agent-key-hint');
|
||||
hint.textContent=r.api_key_masked?('已配置 '+r.api_key_masked):'未配置 Key';
|
||||
});
|
||||
}
|
||||
function saveAgentConfig(){
|
||||
const body={api_base:document.getElementById('agent-api-base').value.trim(),
|
||||
model:document.getElementById('agent-model').value.trim(),
|
||||
default_serial:document.getElementById('agent-default-serial').value.trim()};
|
||||
const key=document.getElementById('agent-api-key').value.trim();
|
||||
if(key)body.api_key=key;
|
||||
apiPost('/api/agent/config',body).then(r=>{
|
||||
if(r&&r.ok){showToast('配置已保存','success');loadAgentConfig();}
|
||||
else showToast('保存失败: '+((r&&r.error)||''),'error');
|
||||
});
|
||||
}
|
||||
function runAgent(){
|
||||
const prompt=document.getElementById('agent-prompt').value.trim();
|
||||
if(!prompt){showToast('请输入指令','error');return;}
|
||||
apiPost('/api/agent/run',{prompt}).then(r=>{
|
||||
if(r&&r.ok){
|
||||
showToast('Agent 已启动','success');
|
||||
document.getElementById('agent-status').textContent='运行中...';
|
||||
document.getElementById('btn-agent-run').disabled=true;
|
||||
document.getElementById('agent-answer-wrap').style.display='none';
|
||||
startAgentPoll();
|
||||
}else showToast('启动失败: '+((r&&r.error)||''),'error');
|
||||
});
|
||||
}
|
||||
function startAgentPoll(){
|
||||
if(_agentPoll)clearInterval(_agentPoll);
|
||||
_agentPoll=setInterval(pollAgent,2000);
|
||||
pollAgent();
|
||||
}
|
||||
function stopAgentPoll(){
|
||||
if(_agentPoll){clearInterval(_agentPoll);_agentPoll=null;}
|
||||
}
|
||||
function pollAgent(){
|
||||
apiGet('/api/agent/status').then(r=>{
|
||||
if(!r||!r.ok)return;
|
||||
const st=document.getElementById('agent-status');
|
||||
if(r.state==='running'){
|
||||
st.textContent='运行中... ('+(r.steps||[]).length+' 步)';
|
||||
}else{
|
||||
st.textContent=r.state==='done'?'完成':'失败';
|
||||
document.getElementById('btn-agent-run').disabled=false;
|
||||
stopAgentPoll();
|
||||
}
|
||||
// 步骤流(含截图缩略)
|
||||
const steps=document.getElementById('agent-steps');
|
||||
const html=(r.steps||[]).map(s=>{
|
||||
const args=esc(s.args||'');
|
||||
const img=s.image
|
||||
?'<img src="data:image/jpeg;base64,'+s.image+'" style="max-width:120px;border-radius:6px;margin-top:4px;display:block">':'';
|
||||
return '<div style="padding:6px 0;border-bottom:1px solid var(--ink-line)">'
|
||||
+'<code style="color:var(--teal)">'+esc(s.tool||'')+'</code> '
|
||||
+'<span class="text-muted">'+args+'</span>'+img+'</div>';
|
||||
}).join('')||'';
|
||||
steps.innerHTML=html;
|
||||
// 最终回答
|
||||
if(r.answer){
|
||||
const wrap=document.getElementById('agent-answer-wrap');
|
||||
wrap.style.display='block';
|
||||
document.getElementById('agent-answer').textContent=r.answer;
|
||||
}
|
||||
if(r.error){
|
||||
st.textContent='错误: '+r.error;
|
||||
document.getElementById('btn-agent-run').disabled=false;
|
||||
stopAgentPoll();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ================== 设备自动发现(扫描 → 待连接池 → 确认) ==================
|
||||
function loadDiscovery(){
|
||||
apiGet('/api/devices/discovery').then(r=>{
|
||||
|
||||
@@ -303,6 +303,27 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
|
||||
.el-picker-item .ep-info .ep-attrs{color:var(--text-light);font-size:11px;margin-top:2px;word-break:break-all}
|
||||
.el-picker-item .ep-info .ep-attrs span{margin-right:8px}
|
||||
.el-picker-item .ep-sel{font-size:10px;color:#fff;background:var(--primary);padding:1px 7px;border-radius:99px;flex-shrink:0;font-weight:600}
|
||||
|
||||
/* ===== AI 控制台(聊天界面) ===== */
|
||||
.agent-shell{display:flex;flex-direction:column;height:calc(100vh - 150px);min-height:480px;border:1px solid var(--card-line);border-radius:14px;overflow:hidden;background:#0d1117}
|
||||
.agent-topbar{display:flex;align-items:center;justify-content:space-between;padding:10px 16px;border-bottom:1px solid var(--ink-line);background:#11151c}
|
||||
.agent-title{font-size:14px;font-weight:700}
|
||||
.agent-model-tag{margin-left:8px;font-size:11px;color:var(--teal);font-family:var(--mono);background:rgba(45,212,191,.08);padding:2px 8px;border-radius:10px}
|
||||
.agent-running-tag{color:#f87171;font-size:12px;margin-right:8px}
|
||||
.agent-top-actions{display:flex;align-items:center;gap:6px}
|
||||
.agent-chat{flex:1;overflow-y:auto;padding:18px 22px;display:flex;flex-direction:column;gap:14px}
|
||||
.agent-msg{max-width:78%;padding:10px 14px;border-radius:12px;font-size:13.5px;line-height:1.65;white-space:pre-wrap;word-break:break-word}
|
||||
.agent-msg.user{align-self:flex-end;background:#155e63;color:#e6fffb;border-bottom-right-radius:3px}
|
||||
.agent-msg.assistant{align-self:flex-start;background:#161c26;border:1px solid var(--card-line);border-bottom-left-radius:3px;color:var(--text)}
|
||||
.agent-msg .reasoning{color:#8b95a7;font-size:12.5px;border-left:2px solid #2b3441;padding-left:8px;margin:6px 0}
|
||||
.agent-toolcards{display:flex;flex-direction:column;gap:4px;margin-bottom:8px}
|
||||
.agent-toolcard{display:flex;align-items:center;gap:8px;background:#0d1117;border:1px solid #232936;border-radius:8px;padding:5px 10px;font-size:12px;font-family:var(--mono);flex-wrap:wrap}
|
||||
.agent-toolcard .dot{width:7px;height:7px;border-radius:50%;background:var(--teal);flex:none}
|
||||
.agent-toolcard img{max-width:130px;border-radius:6px;margin-top:3px}
|
||||
.agent-inputbar{display:flex;gap:10px;padding:12px 14px;border-top:1px solid var(--ink-line);background:#11151c}
|
||||
.agent-inputbar textarea{flex:1;resize:none;background:#0d1117;border:1px solid var(--card-line);border-radius:10px;padding:9px 12px;color:var(--text);font-size:13.5px;outline:none;line-height:1.5}
|
||||
.agent-inputbar textarea:focus{border-color:var(--teal)}
|
||||
.agent-empty{color:#4b5563;font-size:13px;text-align:center;padding:40px 0}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
@@ -316,6 +337,7 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
|
||||
<button class="tab" data-tab="logs" onclick="showTab('logs')" data-perm="logs">日志</button>
|
||||
<button class="tab" data-tab="users" onclick="showTab('users')" data-perm="admin">用户</button>
|
||||
<button class="tab" data-tab="tools" onclick="showTab('tools')" data-perm="admin">工具</button>
|
||||
<button class="tab" data-tab="agent" onclick="showTab('agent')" data-perm="admin">AI 控制台</button>
|
||||
</div>
|
||||
<div class="user-info">
|
||||
<span id="me-username">...</span>
|
||||
@@ -422,6 +444,45 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
|
||||
</div>
|
||||
|
||||
<!-- ========== Tab 3: 分组 ========== -->
|
||||
<!-- ========== Tab: AI 控制台(DeepSeek 风格聊天 + 实时 MCP 步骤) ========== -->
|
||||
<div id="tab-agent" class="tab-panel">
|
||||
<div class="agent-shell">
|
||||
<div class="agent-topbar">
|
||||
<div class="agent-title">🤖 AI 控制台<span id="agent-model-tag" class="agent-model-tag"></span></div>
|
||||
<div class="agent-top-actions">
|
||||
<span id="agent-running-tag" class="agent-running-tag" style="display:none">● 运行中</span>
|
||||
<button class="btn btn-xs" onclick="clearAgentChat()">清空对话</button>
|
||||
<button class="btn btn-xs" onclick="openAgentConfig()">⚙ 配置</button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="agent-chat" class="agent-chat"></div>
|
||||
<div class="agent-inputbar">
|
||||
<textarea id="agent-input" rows="2"
|
||||
placeholder="给 AI 下达指令,例如:打开抖音搜索奚学东,把第一个视频标题告诉我…(Enter 发送,Shift+Enter 换行)"></textarea>
|
||||
<button class="btn btn-primary" id="btn-agent-send" onclick="sendAgentMsg()">发送</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ========== 配置模态框 ========== -->
|
||||
<div id="agent-cfg-overlay" style="display:none;position:fixed;inset:0;background:rgba(0,0,0,.6);z-index:1300;align-items:center;justify-content:center">
|
||||
<div style="background:#11151c;border:1px solid var(--card-line);border-radius:14px;padding:22px;width:460px;max-width:92vw">
|
||||
<div style="font-size:15px;font-weight:700;margin-bottom:14px">模型配置(OpenAI 兼容 API)</div>
|
||||
<div class="form-group"><label>API Base</label>
|
||||
<input type="text" id="acfg-base" class="form-control" placeholder="https://api.deepseek.com" style="font-family:monospace"></div>
|
||||
<div class="form-group"><label>模型名</label>
|
||||
<input type="text" id="acfg-model" class="form-control" placeholder="deepseek-v4-flash-vision-exp"></div>
|
||||
<div class="form-group"><label>API Key <span class="text-muted" id="acfg-key-hint"></span></label>
|
||||
<input type="password" id="acfg-key" class="form-control" placeholder="留空则不修改"></div>
|
||||
<div class="form-group"><label>默认设备 serial(可选)</label>
|
||||
<input type="text" id="acfg-serial" class="form-control" placeholder="192.168.20.66:5555" style="font-family:monospace"></div>
|
||||
<div style="display:flex;justify-content:flex-end;gap:8px;margin-top:6px">
|
||||
<button class="btn" onclick="closeAgentConfig()">取消</button>
|
||||
<button class="btn btn-primary" onclick="saveAgentConfig()">保存</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ========== Tab 4: 日志 ========== -->
|
||||
<div id="tab-logs" class="tab-panel">
|
||||
<div class="page-title">日志查看</div>
|
||||
@@ -480,7 +541,6 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
|
||||
<button class="sub-tab" data-sub="devapps" onclick="showSubTab('tools','devapps')">设备已装应用</button>
|
||||
<button class="sub-tab" data-sub="devpool" onclick="showSubTab('tools','devpool')">设备池管理</button>
|
||||
<button class="sub-tab" data-sub="groups" onclick="showSubTab('tools','groups')">设备分组</button>
|
||||
<button class="sub-tab" data-sub="agent" onclick="showSubTab('tools','agent')">AI 控制台</button>
|
||||
</div>
|
||||
|
||||
<!-- 子分栏 1:剪贴板注入 -->
|
||||
@@ -703,36 +763,7 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
|
||||
<div class="help">管理设备分组,用于任务按分组执行。分组内的设备需先在<b>设备池管理</b>中确认(不在池内的设备不参与调度)。</div>
|
||||
</div>
|
||||
|
||||
<!-- 子分栏:AI 控制台(第三方模型经 Agent 控制手机) -->
|
||||
<div class="sub-panel" id="tools-sub-agent">
|
||||
<details style="margin-bottom:10px">
|
||||
<summary style="cursor:pointer;color:var(--muted)">⚙ 模型配置(OpenAI 兼容 API)</summary>
|
||||
<div style="padding:10px 0 4px;display:flex;flex-wrap:wrap;gap:8px;align-items:center">
|
||||
<input type="text" id="agent-api-base" class="form-control" style="width:260px;font-family:monospace"
|
||||
placeholder="API Base,如 https://api.deepseek.com">
|
||||
<input type="text" id="agent-model" class="form-control" style="width:220px"
|
||||
placeholder="模型名,如 deepseek-v4-flash-vision-exp">
|
||||
<input type="password" id="agent-api-key" class="form-control" style="width:260px"
|
||||
placeholder="API Key(留空不改)">
|
||||
<input type="text" id="agent-default-serial" class="form-control" style="width:220px;font-family:monospace"
|
||||
placeholder="默认设备 serial(可选)">
|
||||
<button class="btn btn-primary" onclick="saveAgentConfig()">保存配置</button>
|
||||
<span id="agent-key-hint" class="text-muted"></span>
|
||||
</div>
|
||||
</details>
|
||||
<div class="toolbar" style="flex-wrap:wrap">
|
||||
<textarea id="agent-prompt" class="form-control" rows="2" style="flex:1;min-width:300px"
|
||||
placeholder="指令:如「打开抖音搜索奚学东,截个图描述结果」"></textarea>
|
||||
<button class="btn btn-primary" id="btn-agent-run" onclick="runAgent()">▶ 执行</button>
|
||||
<span id="agent-status" class="text-muted"></span>
|
||||
</div>
|
||||
<div id="agent-steps" style="margin-top:10px"></div>
|
||||
<div id="agent-answer-wrap" style="display:none;margin-top:12px">
|
||||
<div class="section-title">回答</div>
|
||||
<div id="agent-answer" style="white-space:pre-wrap;line-height:1.7;background:#0d1117;border:1px solid var(--card-line);border-radius:10px;padding:12px"></div>
|
||||
</div>
|
||||
<div class="help">AI 通过截图观察屏幕并执行操作(元素定位优先,坐标兜底)。模型与 Key 配置保存在平台(管理员可见),请勿填他人密钥。</div>
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
@@ -798,5 +829,6 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
|
||||
<script src="/static/admin/tools.js"></script>
|
||||
<script src="/static/admin/apps.js"></script>
|
||||
<script src="/static/admin/admin.js"></script>
|
||||
<script src="/static/admin/agent.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
+70
-29
@@ -1,17 +1,27 @@
|
||||
"""AI 控制台 API:Agent 配置(模型/Key 前端可配)+ 运行 + 状态轮询。
|
||||
"""AI 控制台 API:模型/Key 前端配置 + Agent 流式执行(SSE 事件流)。
|
||||
|
||||
Agent 在平台进程内跑(后台线程),连本机 MCP Server(8033)执行工具。
|
||||
单实例:同一时间只允许一个 Agent 运行,避免多路操作设备冲突。
|
||||
流程:
|
||||
POST /api/agent/run {prompt} → 启动 Agent 线程,返回 run_id
|
||||
GET /api/agent/stream?run_id= → SSE 事件流(EventSource 订阅):
|
||||
event: delta {text, kind: content|reasoning} 流式文本增量
|
||||
event: step {tool, args, image?} 工具调用完成(MCP 步骤)
|
||||
event: done {answer} 完成
|
||||
event: error {message} 失败
|
||||
GET/POST /api/agent/config → 配置读写(key 打码回显)
|
||||
|
||||
单实例:同时只允许一个 Agent 运行。
|
||||
"""
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
|
||||
from flask import Blueprint, jsonify, request
|
||||
from flask import Blueprint, Response, jsonify, request
|
||||
|
||||
from core.logger import get_logger
|
||||
from core.models import db
|
||||
@@ -26,9 +36,10 @@ _CFG_KEYS = {"api_base": "agent_api_base",
|
||||
"api_key": "agent_api_key",
|
||||
"default_serial": "agent_default_serial"}
|
||||
|
||||
# ---------- 运行状态(单实例) ----------
|
||||
# ---------- 运行状态(单实例 + 事件队列) ----------
|
||||
_run = {"id": None, "state": "idle", "prompt": "", "serial": "",
|
||||
"steps": [], "answer": "", "error": ""}
|
||||
"answer": "", "error": ""}
|
||||
_queues = {} # run_id -> queue.Queue(SSE 消费者读取)
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
@@ -47,6 +58,7 @@ def _read_cfg():
|
||||
return {k: _meta_get(v) for k, v in _CFG_KEYS.items()}
|
||||
|
||||
|
||||
# ================== 配置 ==================
|
||||
@bp.route("/api/agent/config", methods=["GET"])
|
||||
@admin_required
|
||||
def agent_config_get():
|
||||
@@ -70,10 +82,11 @@ def agent_config_save():
|
||||
return jsonify({"ok": True, "msg": "已保存"})
|
||||
|
||||
|
||||
# ================== 运行 ==================
|
||||
@bp.route("/api/agent/run", methods=["POST"])
|
||||
@admin_required
|
||||
def agent_run():
|
||||
"""启动 Agent 执行指令:{prompt, serial?}。运行中返回 409。"""
|
||||
"""启动 Agent:{prompt, serial?}。运行中返回 409。"""
|
||||
data = request.json or {}
|
||||
prompt = (data.get("prompt") or "").strip()
|
||||
if not prompt:
|
||||
@@ -81,29 +94,53 @@ def agent_run():
|
||||
cfg = _read_cfg()
|
||||
if not cfg.get("api_key"):
|
||||
return jsonify({"ok": False, "error": "请先在配置区填写 API Key"}), 400
|
||||
if not cfg.get("model"):
|
||||
return jsonify({"ok": False, "error": "请先填写模型名"}), 400
|
||||
with _lock:
|
||||
if _run["state"] == "running":
|
||||
return jsonify({"ok": False, "error": "已有 Agent 运行中,请等待完成"}), 409
|
||||
run_id = uuid.uuid4().hex[:8]
|
||||
_run.update(id=run_id, state="running", prompt=prompt,
|
||||
serial=(data.get("serial") or "").strip(),
|
||||
steps=[], answer="", error="")
|
||||
_log.info(f"Agent 启动: {prompt[:60]} serial={_run['serial']}")
|
||||
answer="", error="")
|
||||
_queues[run_id] = queue.Queue()
|
||||
_log.info(f"Agent 启动: {prompt[:60]}")
|
||||
threading.Thread(target=_agent_thread, args=(run_id, prompt, cfg),
|
||||
daemon=True).start()
|
||||
return jsonify({"ok": True, "run_id": run_id})
|
||||
|
||||
|
||||
@bp.route("/api/agent/status")
|
||||
@bp.route("/api/agent/stream")
|
||||
@admin_required
|
||||
def agent_status():
|
||||
"""Agent 运行状态(前端轮询):{state, prompt, serial, steps, answer, error}。"""
|
||||
def agent_stream():
|
||||
"""SSE 事件流(EventSource):delta/step/done/error。"""
|
||||
run_id = request.args.get("run_id", "")
|
||||
with _lock:
|
||||
return jsonify({"ok": True, **_run})
|
||||
if run_id != _run["id"]:
|
||||
return jsonify({"ok": False, "error": "run_id 不存在"}), 404
|
||||
q = _queues.get(run_id)
|
||||
|
||||
def gen():
|
||||
while True:
|
||||
try:
|
||||
evt = q.get(timeout=15)
|
||||
except queue.Empty:
|
||||
yield ": keepalive\n\n" # 心跳防超时
|
||||
continue
|
||||
if evt is None:
|
||||
break
|
||||
kind, payload = evt
|
||||
yield f"event: {kind}\ndata: {json.dumps(payload, ensure_ascii=False)}\n\n"
|
||||
if kind in ("done", "error"):
|
||||
break
|
||||
|
||||
return Response(gen(), mimetype="text/event-stream",
|
||||
headers={"Cache-Control": "no-cache",
|
||||
"X-Accel-Buffering": "no"})
|
||||
|
||||
|
||||
def _shrink_image(b64, width=220, quality=50):
|
||||
"""截图降采样(web 展示用,避免大图撑爆轮询响应)。失败原样返回。"""
|
||||
"""截图降采样(SSE step 事件用,控制传输体积)。失败原样返回。"""
|
||||
try:
|
||||
from PIL import Image
|
||||
img = Image.open(io.BytesIO(base64.b64decode(b64)))
|
||||
@@ -117,42 +154,46 @@ def _shrink_image(b64, width=220, quality=50):
|
||||
|
||||
|
||||
def _agent_thread(run_id, prompt, cfg):
|
||||
"""后台线程:跑 Agent,回调记录步骤。"""
|
||||
"""后台线程:Agent 流式执行,事件推入队列供 SSE 消费。"""
|
||||
q = _queues.get(run_id)
|
||||
try:
|
||||
# 平台进程 cwd=/app(含 mcp_agent 包),进程内 import
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from mcp_agent.agent import Agent
|
||||
|
||||
def on_step(step):
|
||||
with _lock:
|
||||
if _run["id"] != run_id:
|
||||
return
|
||||
rec = {"tool": step.get("tool"), "args": str(step.get("args"))[:120]}
|
||||
if step.get("image_b64"):
|
||||
rec["image"] = _shrink_image(step["image_b64"])
|
||||
# 保留最近 6 步截图,防止响应过大
|
||||
_run["steps"] = _run["steps"][-5:]
|
||||
_run["steps"].append(rec)
|
||||
def on_delta(text, kind):
|
||||
q.put(("delta", {"text": text, "kind": kind}))
|
||||
|
||||
def on_tool(step):
|
||||
rec = {"tool": step.get("tool"),
|
||||
"args": str(step.get("args"))[:200]}
|
||||
if step.get("image"):
|
||||
rec["image"] = _shrink_image(step["image"])
|
||||
q.put(("step", rec))
|
||||
|
||||
agent = Agent()
|
||||
# 用 web 配置覆盖 Agent 默认(api key 等由前端配置)
|
||||
agent.s.api_base = cfg.get("api_base") or agent.s.api_base
|
||||
agent.s.model = cfg.get("model") or agent.s.model
|
||||
agent.s.api_key = cfg.get("api_key") or agent.s.api_key
|
||||
agent.s.default_serial = cfg.get("default_serial") or agent.s.default_serial
|
||||
agent.on_step = on_step
|
||||
|
||||
async def _execute():
|
||||
await agent._load_tools()
|
||||
return await agent.run(prompt, cfg.get("default_serial") or "")
|
||||
return await agent.run_stream(prompt, cfg.get("default_serial") or "",
|
||||
on_delta=on_delta, on_tool=on_tool)
|
||||
|
||||
# 整体超时保护:模型/工具卡死时结束运行,避免单实例被永久占用
|
||||
# 整体超时保护:卡死时结束,释放单实例
|
||||
answer = asyncio.run(asyncio.wait_for(_execute(), timeout=600))
|
||||
with _lock:
|
||||
_run["state"] = "done"
|
||||
_run["answer"] = answer
|
||||
q.put(("done", {"answer": answer}))
|
||||
except Exception as e:
|
||||
_log.warning(f"Agent 运行异常: {e}")
|
||||
with _lock:
|
||||
_run["state"] = "error"
|
||||
_run["error"] = f"{type(e).__name__}: {str(e)[:200]}"
|
||||
q.put(("error", {"message": str(e)[:200]}))
|
||||
finally:
|
||||
q.put(None) # 关闭 SSE
|
||||
_queues.pop(run_id, None)
|
||||
|
||||
Reference in New Issue
Block a user