feat(M2): Web AI 控制台——工具页新增子分栏：模型/Key/设备前端可配置（app_meta 存储，key 打码回显），指令执行 + 步骤流轮询（含截图缩略）+ 回答展示；Agent 加 on_step 回调、后台线程单实例运行
This commit is contained in:
+12
-1
@@ -39,6 +39,7 @@ class Agent:
|
||||
self.tools_schema = [] # OpenAI function schema
|
||||
self._tool_exec = {} # name -> callable
|
||||
self.messages = []
|
||||
self.on_step = None # 可选回调 fn(step_dict)，web 展示进度用
|
||||
|
||||
# ---------- MCP 工具桥 ----------
|
||||
async def _load_tools(self):
|
||||
@@ -93,12 +94,22 @@ class Agent:
|
||||
except Exception as e:
|
||||
return {"ok": False, "error": f"工具执行失败: {e}"}, None
|
||||
# de_screenshot：图像分离（作为 image_url 追加给模型看）
|
||||
image_b64 = None
|
||||
if name == "de_screenshot" and isinstance(data, dict) and data.get("ok"):
|
||||
img = (data.get("data") or {}).get("image") or {}
|
||||
if img.get("data"):
|
||||
text_result = {k: v for k, v in (data.get("data") or {}).items()
|
||||
if k != "image"}
|
||||
return text_result, img["data"]
|
||||
image_b64 = img["data"]
|
||||
if self.on_step:
|
||||
try:
|
||||
self.on_step({"tool": name, "args": args,
|
||||
"result": text_result if image_b64 else data,
|
||||
"image_b64": image_b64})
|
||||
except Exception:
|
||||
pass
|
||||
if image_b64:
|
||||
return text_result, image_b64
|
||||
return data, None
|
||||
|
||||
# ---------- 主循环 ----------
|
||||
|
||||
@@ -114,6 +114,7 @@ function showSubTab(tabId, name){
|
||||
tab.querySelectorAll('.sub-tab').forEach(b=>b.classList.toggle('active', b.dataset.sub===name));
|
||||
tab.querySelectorAll('.sub-panel').forEach(p=>p.classList.toggle('active', p.id===tabId+'-sub-'+name));
|
||||
if(name==='groups' && typeof loadGroups==='function') loadGroups();
|
||||
if(name==='agent' && typeof loadAgentConfig==='function') loadAgentConfig();
|
||||
if(name==='devpool' && typeof loadDevPool==='function'){
|
||||
loadDevPool();
|
||||
if(typeof loadDiscovery==='function'){
|
||||
|
||||
@@ -41,6 +41,89 @@ async function loadToolsDevices(force){
|
||||
status.textContent = '共 '+_clipDevices.length+' 台设备';
|
||||
}
|
||||
|
||||
// ================== AI 控制台（模型配置 + 指令执行 + 进度轮询） ==================
|
||||
let _agentPoll = null;
|
||||
|
||||
function loadAgentConfig(){
|
||||
apiGet('/api/agent/config').then(r=>{
|
||||
if(!r||!r.ok)return;
|
||||
const b=document.getElementById('agent-api-base');
|
||||
if(!b)return; // 面板未渲染
|
||||
b.value=r.api_base||'https://api.deepseek.com';
|
||||
document.getElementById('agent-model').value=r.model||'';
|
||||
document.getElementById('agent-default-serial').value=r.default_serial||'';
|
||||
const hint=document.getElementById('agent-key-hint');
|
||||
hint.textContent=r.api_key_masked?('已配置 '+r.api_key_masked):'未配置 Key';
|
||||
});
|
||||
}
|
||||
function saveAgentConfig(){
|
||||
const body={api_base:document.getElementById('agent-api-base').value.trim(),
|
||||
model:document.getElementById('agent-model').value.trim(),
|
||||
default_serial:document.getElementById('agent-default-serial').value.trim()};
|
||||
const key=document.getElementById('agent-api-key').value.trim();
|
||||
if(key)body.api_key=key;
|
||||
apiPost('/api/agent/config',body).then(r=>{
|
||||
if(r&&r.ok){showToast('配置已保存','success');loadAgentConfig();}
|
||||
else showToast('保存失败: '+((r&&r.error)||''),'error');
|
||||
});
|
||||
}
|
||||
function runAgent(){
|
||||
const prompt=document.getElementById('agent-prompt').value.trim();
|
||||
if(!prompt){showToast('请输入指令','error');return;}
|
||||
apiPost('/api/agent/run',{prompt}).then(r=>{
|
||||
if(r&&r.ok){
|
||||
showToast('Agent 已启动','success');
|
||||
document.getElementById('agent-status').textContent='运行中...';
|
||||
document.getElementById('btn-agent-run').disabled=true;
|
||||
document.getElementById('agent-answer-wrap').style.display='none';
|
||||
startAgentPoll();
|
||||
}else showToast('启动失败: '+((r&&r.error)||''),'error');
|
||||
});
|
||||
}
|
||||
function startAgentPoll(){
|
||||
if(_agentPoll)clearInterval(_agentPoll);
|
||||
_agentPoll=setInterval(pollAgent,2000);
|
||||
pollAgent();
|
||||
}
|
||||
function stopAgentPoll(){
|
||||
if(_agentPoll){clearInterval(_agentPoll);_agentPoll=null;}
|
||||
}
|
||||
function pollAgent(){
|
||||
apiGet('/api/agent/status').then(r=>{
|
||||
if(!r||!r.ok)return;
|
||||
const st=document.getElementById('agent-status');
|
||||
if(r.state==='running'){
|
||||
st.textContent='运行中... ('+(r.steps||[]).length+' 步)';
|
||||
}else{
|
||||
st.textContent=r.state==='done'?'完成':'失败';
|
||||
document.getElementById('btn-agent-run').disabled=false;
|
||||
stopAgentPoll();
|
||||
}
|
||||
// 步骤流（含截图缩略）
|
||||
const steps=document.getElementById('agent-steps');
|
||||
const html=(r.steps||[]).map(s=>{
|
||||
const args=esc(s.args||'');
|
||||
const img=s.image
|
||||
?'<img src="data:image/jpeg;base64,'+s.image+'" style="max-width:120px;border-radius:6px;margin-top:4px;display:block">':'';
|
||||
return '<div style="padding:6px 0;border-bottom:1px solid var(--ink-line)">'
|
||||
+'<code style="color:var(--teal)">'+esc(s.tool||'')+'</code> '
|
||||
+'<span class="text-muted">'+args+'</span>'+img+'</div>';
|
||||
}).join('')||'';
|
||||
steps.innerHTML=html;
|
||||
// 最终回答
|
||||
if(r.answer){
|
||||
const wrap=document.getElementById('agent-answer-wrap');
|
||||
wrap.style.display='block';
|
||||
document.getElementById('agent-answer').textContent=r.answer;
|
||||
}
|
||||
if(r.error){
|
||||
st.textContent='错误: '+r.error;
|
||||
document.getElementById('btn-agent-run').disabled=false;
|
||||
stopAgentPoll();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ================== 设备自动发现（扫描 → 待连接池 → 确认） ==================
|
||||
function loadDiscovery(){
|
||||
apiGet('/api/devices/discovery').then(r=>{
|
||||
|
||||
@@ -480,6 +480,7 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
|
||||
<button class="sub-tab" data-sub="devapps" onclick="showSubTab('tools','devapps')">设备已装应用</button>
|
||||
<button class="sub-tab" data-sub="devpool" onclick="showSubTab('tools','devpool')">设备池管理</button>
|
||||
<button class="sub-tab" data-sub="groups" onclick="showSubTab('tools','groups')">设备分组</button>
|
||||
<button class="sub-tab" data-sub="agent" onclick="showSubTab('tools','agent')">AI 控制台</button>
|
||||
</div>
|
||||
|
||||
<!-- 子分栏 1：剪贴板注入 -->
|
||||
@@ -702,6 +703,37 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
|
||||
<div class="help">管理设备分组，用于任务按分组执行。分组内的设备需先在<b>设备池管理</b>中确认（不在池内的设备不参与调度）。</div>
|
||||
</div>
|
||||
|
||||
<!-- 子分栏：AI 控制台（第三方模型经 Agent 控制手机） -->
|
||||
<div class="sub-panel" id="tools-sub-agent">
|
||||
<details style="margin-bottom:10px">
|
||||
<summary style="cursor:pointer;color:var(--muted)">⚙ 模型配置（OpenAI 兼容 API）</summary>
|
||||
<div style="padding:10px 0 4px;display:flex;flex-wrap:wrap;gap:8px;align-items:center">
|
||||
<input type="text" id="agent-api-base" class="form-control" style="width:260px;font-family:monospace"
|
||||
placeholder="API Base，如 https://api.deepseek.com">
|
||||
<input type="text" id="agent-model" class="form-control" style="width:220px"
|
||||
placeholder="模型名，如 deepseek-v4-flash-vision-exp">
|
||||
<input type="password" id="agent-api-key" class="form-control" style="width:260px"
|
||||
placeholder="API Key（留空不改）">
|
||||
<input type="text" id="agent-default-serial" class="form-control" style="width:220px;font-family:monospace"
|
||||
placeholder="默认设备 serial（可选）">
|
||||
<button class="btn btn-primary" onclick="saveAgentConfig()">保存配置</button>
|
||||
<span id="agent-key-hint" class="text-muted"></span>
|
||||
</div>
|
||||
</details>
|
||||
<div class="toolbar" style="flex-wrap:wrap">
|
||||
<textarea id="agent-prompt" class="form-control" rows="2" style="flex:1;min-width:300px"
|
||||
placeholder="指令：如「打开抖音搜索奚学东，截个图描述结果」"></textarea>
|
||||
<button class="btn btn-primary" id="btn-agent-run" onclick="runAgent()">▶ 执行</button>
|
||||
<span id="agent-status" class="text-muted"></span>
|
||||
</div>
|
||||
<div id="agent-steps" style="margin-top:10px"></div>
|
||||
<div id="agent-answer-wrap" style="display:none;margin-top:12px">
|
||||
<div class="section-title">回答</div>
|
||||
<div id="agent-answer" style="white-space:pre-wrap;line-height:1.7;background:#0d1117;border:1px solid var(--card-line);border-radius:10px;padding:12px"></div>
|
||||
</div>
|
||||
<div class="help">AI 通过截图观察屏幕并执行操作（元素定位优先，坐标兜底）。模型与 Key 配置保存在平台（管理员可见），请勿填他人密钥。</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
</div><!-- /.content -->
|
||||
|
||||
+2
-1
@@ -23,6 +23,7 @@ def register_blueprints(app):
|
||||
from .devices_api import bp as devices_bp
|
||||
from .apks_api import bp as apks_bp
|
||||
from .tailscale_api import bp as tailscale_bp
|
||||
from .agent_api import bp as agent_bp
|
||||
for bp in (auth_bp, monitor_bp, tasks_bp, admin_bp, tools_bp,
|
||||
devices_bp, apks_bp, tailscale_bp):
|
||||
devices_bp, apks_bp, tailscale_bp, agent_bp):
|
||||
app.register_blueprint(bp)
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
"""AI 控制台 API：Agent 配置（模型/Key 前端可配）+ 运行 + 状态轮询。
|
||||
|
||||
Agent 在平台进程内跑（后台线程），连本机 MCP Server（8033）执行工具。
|
||||
单实例：同一时间只允许一个 Agent 运行，避免多路操作设备冲突。
|
||||
"""
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
|
||||
from flask import Blueprint, jsonify, request
|
||||
|
||||
from core.logger import get_logger
|
||||
from core.models import db
|
||||
from web.auth import admin_required
|
||||
|
||||
_log = get_logger("web.agent")
|
||||
bp = Blueprint("agent", __name__)
|
||||
|
||||
# ---------- 配置键（app_meta） ----------
|
||||
_CFG_KEYS = {"api_base": "agent_api_base",
|
||||
"model": "agent_model",
|
||||
"api_key": "agent_api_key",
|
||||
"default_serial": "agent_default_serial"}
|
||||
|
||||
# ---------- 运行状态（单实例） ----------
|
||||
_run = {"id": None, "state": "idle", "prompt": "", "serial": "",
|
||||
"steps": [], "answer": "", "error": ""}
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def _meta_get(key):
|
||||
return db.session.execute(
|
||||
db.text("SELECT value FROM app_meta WHERE key=:k"), {"k": key}).scalar() or ""
|
||||
|
||||
|
||||
def _meta_put(key, value):
|
||||
db.session.execute(
|
||||
db.text("INSERT OR REPLACE INTO app_meta(key,value) VALUES(:k,:v)"),
|
||||
{"k": key, "v": str(value)})
|
||||
|
||||
|
||||
def _read_cfg():
|
||||
return {k: _meta_get(v) for k, v in _CFG_KEYS.items()}
|
||||
|
||||
|
||||
@bp.route("/api/agent/config", methods=["GET"])
|
||||
@admin_required
|
||||
def agent_config_get():
|
||||
"""读 Agent 配置（key 打码返回）。"""
|
||||
cfg = _read_cfg()
|
||||
if cfg["api_key"]:
|
||||
k = cfg["api_key"]
|
||||
cfg["api_key_masked"] = k[:6] + "***" + k[-4:]
|
||||
return jsonify({"ok": True, **cfg})
|
||||
|
||||
|
||||
@bp.route("/api/agent/config", methods=["POST"])
|
||||
@admin_required
|
||||
def agent_config_save():
|
||||
"""保存 Agent 配置：{api_base?, model?, api_key?, default_serial?} 部分更新。"""
|
||||
data = request.json or {}
|
||||
for key, meta_key in _CFG_KEYS.items():
|
||||
if key in data and data[key] is not None:
|
||||
_meta_put(meta_key, str(data[key]).strip())
|
||||
db.session.commit()
|
||||
return jsonify({"ok": True, "msg": "已保存"})
|
||||
|
||||
|
||||
@bp.route("/api/agent/run", methods=["POST"])
|
||||
@admin_required
|
||||
def agent_run():
|
||||
"""启动 Agent 执行指令：{prompt, serial?}。运行中返回 409。"""
|
||||
data = request.json or {}
|
||||
prompt = (data.get("prompt") or "").strip()
|
||||
if not prompt:
|
||||
return jsonify({"ok": False, "error": "请输入指令"}), 400
|
||||
cfg = _read_cfg()
|
||||
if not cfg.get("api_key"):
|
||||
return jsonify({"ok": False, "error": "请先在配置区填写 API Key"}), 400
|
||||
with _lock:
|
||||
if _run["state"] == "running":
|
||||
return jsonify({"ok": False, "error": "已有 Agent 运行中，请等待完成"}), 409
|
||||
run_id = uuid.uuid4().hex[:8]
|
||||
_run.update(id=run_id, state="running", prompt=prompt,
|
||||
serial=(data.get("serial") or "").strip(),
|
||||
steps=[], answer="", error="")
|
||||
_log.info(f"Agent 启动: {prompt[:60]} serial={_run['serial']}")
|
||||
threading.Thread(target=_agent_thread, args=(run_id, prompt, cfg),
|
||||
daemon=True).start()
|
||||
return jsonify({"ok": True, "run_id": run_id})
|
||||
|
||||
|
||||
@bp.route("/api/agent/status")
|
||||
@admin_required
|
||||
def agent_status():
|
||||
"""Agent 运行状态（前端轮询）：{state, prompt, serial, steps, answer, error}。"""
|
||||
with _lock:
|
||||
return jsonify({"ok": True, **_run})
|
||||
|
||||
|
||||
def _shrink_image(b64, width=220, quality=50):
|
||||
"""截图降采样（web 展示用，避免大图撑爆轮询响应）。失败原样返回。"""
|
||||
try:
|
||||
from PIL import Image
|
||||
img = Image.open(io.BytesIO(base64.b64decode(b64)))
|
||||
if img.width > width:
|
||||
img = img.resize((width, int(img.height * width / img.width)))
|
||||
buf = io.BytesIO()
|
||||
img.convert("RGB").save(buf, "JPEG", quality=quality)
|
||||
return base64.b64encode(buf.getvalue()).decode()
|
||||
except Exception:
|
||||
return b64
|
||||
|
||||
|
||||
def _agent_thread(run_id, prompt, cfg):
|
||||
"""后台线程：跑 Agent，回调记录步骤。"""
|
||||
try:
|
||||
# 平台进程 cwd=/app（含 mcp_agent 包），进程内 import
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from mcp_agent.agent import Agent
|
||||
|
||||
def on_step(step):
|
||||
with _lock:
|
||||
if _run["id"] != run_id:
|
||||
return
|
||||
rec = {"tool": step.get("tool"), "args": str(step.get("args"))[:120]}
|
||||
if step.get("image_b64"):
|
||||
rec["image"] = _shrink_image(step["image_b64"])
|
||||
# 保留最近 6 步截图，防止响应过大
|
||||
_run["steps"] = _run["steps"][-5:]
|
||||
_run["steps"].append(rec)
|
||||
|
||||
agent = Agent()
|
||||
# 用 web 配置覆盖 Agent 默认（api key 等由前端配置）
|
||||
agent.s.api_base = cfg.get("api_base") or agent.s.api_base
|
||||
agent.s.model = cfg.get("model") or agent.s.model
|
||||
agent.s.api_key = cfg.get("api_key") or agent.s.api_key
|
||||
agent.s.default_serial = cfg.get("default_serial") or agent.s.default_serial
|
||||
agent.on_step = on_step
|
||||
|
||||
async def _run():
|
||||
await agent._load_tools()
|
||||
return await agent.run(prompt, cfg.get("default_serial") or "")
|
||||
|
||||
answer = asyncio.run(_run())
|
||||
with _lock:
|
||||
_run["state"] = "done"
|
||||
_run["answer"] = answer
|
||||
except Exception as e:
|
||||
_log.warning(f"Agent 运行异常: {e}")
|
||||
with _lock:
|
||||
_run["state"] = "error"
|
||||
_run["error"] = f"{type(e).__name__}: {str(e)[:200]}"
|
||||
Reference in New Issue
Block a user