From fea5a57fc33a89acc2b70f24ed5e61463b536296 Mon Sep 17 00:00:00 2001 From: butubb <1422726308@qq.com> Date: Fri, 4 Sep 2026 12:54:17 +0800 Subject: [PATCH] =?UTF-8?q?feat(M2):=20Web=20AI=20=E6=8E=A7=E5=88=B6?= =?UTF-8?q?=E5=8F=B0=E2=80=94=E2=80=94=E5=B7=A5=E5=85=B7=E9=A1=B5=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E5=AD=90=E5=88=86=E6=A0=8F=EF=BC=9A=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?/Key/=E8=AE=BE=E5=A4=87=E5=89=8D=E7=AB=AF=E5=8F=AF=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=EF=BC=88app=5Fmeta=20=E5=AD=98=E5=82=A8=EF=BC=8Ckey?= =?UTF-8?q?=20=E6=89=93=E7=A0=81=E5=9B=9E=E6=98=BE=EF=BC=89=EF=BC=8C?= =?UTF-8?q?=E6=8C=87=E4=BB=A4=E6=89=A7=E8=A1=8C=20+=20=E6=AD=A5=E9=AA=A4?= =?UTF-8?q?=E6=B5=81=E8=BD=AE=E8=AF=A2=EF=BC=88=E5=90=AB=E6=88=AA=E5=9B=BE?= =?UTF-8?q?=E7=BC=A9=E7=95=A5=EF=BC=89+=20=E5=9B=9E=E7=AD=94=E5=B1=95?= =?UTF-8?q?=E7=A4=BA=EF=BC=9BAgent=20=E5=8A=A0=20on=5Fstep=20=E5=9B=9E?= =?UTF-8?q?=E8=B0=83=E3=80=81=E5=90=8E=E5=8F=B0=E7=BA=BF=E7=A8=8B=E5=8D=95?= =?UTF-8?q?=E5=AE=9E=E4=BE=8B=E8=BF=90=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mcp_agent/agent.py | 13 ++- static/admin/base.js | 1 + static/admin/tools.js | 83 ++++++++++++++++++ templates/admin/monitor.html | 32 +++++++ web/__init__.py | 3 +- web/agent_api.py | 157 +++++++++++++++++++++++++++++++++++ 6 files changed, 287 insertions(+), 2 deletions(-) create mode 100644 web/agent_api.py diff --git a/mcp_agent/agent.py b/mcp_agent/agent.py index 826e4a6..09f4661 100644 --- a/mcp_agent/agent.py +++ b/mcp_agent/agent.py @@ -39,6 +39,7 @@ class Agent: self.tools_schema = [] # OpenAI function schema self._tool_exec = {} # name -> callable self.messages = [] + self.on_step = None # 可选回调 fn(step_dict)，web 展示进度用 # ---------- MCP 工具桥 ---------- async def _load_tools(self): @@ -93,12 +94,22 @@ class Agent: except Exception as e: return {"ok": False, "error": f"工具执行失败: {e}"}, None # de_screenshot：图像分离（作为 image_url 追加给模型看） + image_b64 = None if name == "de_screenshot" and isinstance(data, dict) and data.get("ok"): img = (data.get("data") or {}).get("image") or {} if img.get("data"): text_result = {k: v for k, v in (data.get("data") or {}).items() if k != "image"} - return text_result, img["data"] + image_b64 = img["data"] + if self.on_step: + try: + self.on_step({"tool": name, "args": args, + "result": text_result if image_b64 else data, + "image_b64": image_b64}) + except Exception: + pass + if image_b64: + return text_result, image_b64 return data, None # ---------- 主循环 ---------- diff --git a/static/admin/base.js b/static/admin/base.js index 27f6e3d..548f9a6 100644 --- a/static/admin/base.js +++ b/static/admin/base.js @@ -114,6 +114,7 @@ function showSubTab(tabId, name){ tab.querySelectorAll('.sub-tab').forEach(b=>b.classList.toggle('active', b.dataset.sub===name)); tab.querySelectorAll('.sub-panel').forEach(p=>p.classList.toggle('active', p.id===tabId+'-sub-'+name)); if(name==='groups' && typeof loadGroups==='function') loadGroups(); + if(name==='agent' && typeof loadAgentConfig==='function') loadAgentConfig(); if(name==='devpool' && typeof loadDevPool==='function'){ loadDevPool(); if(typeof loadDiscovery==='function'){ diff --git a/static/admin/tools.js b/static/admin/tools.js index d81aaf2..c106461 100644 --- a/static/admin/tools.js +++ b/static/admin/tools.js @@ -41,6 +41,89 @@ async function loadToolsDevices(force){ status.textContent = '共 '+_clipDevices.length+' 台设备'; } +// ================== AI 控制台（模型配置 + 指令执行 + 进度轮询） ================== +let _agentPoll = null; + +function loadAgentConfig(){ + apiGet('/api/agent/config').then(r=>{ + if(!r||!r.ok)return; + const b=document.getElementById('agent-api-base'); + if(!b)return; // 面板未渲染 + b.value=r.api_base||'https://api.deepseek.com'; + document.getElementById('agent-model').value=r.model||''; + document.getElementById('agent-default-serial').value=r.default_serial||''; + const hint=document.getElementById('agent-key-hint'); + hint.textContent=r.api_key_masked?('已配置 '+r.api_key_masked):'未配置 Key'; + }); +} +function saveAgentConfig(){ + const body={api_base:document.getElementById('agent-api-base').value.trim(), + model:document.getElementById('agent-model').value.trim(), + default_serial:document.getElementById('agent-default-serial').value.trim()}; + const key=document.getElementById('agent-api-key').value.trim(); + if(key)body.api_key=key; + apiPost('/api/agent/config',body).then(r=>{ + if(r&&r.ok){showToast('配置已保存','success');loadAgentConfig();} + else showToast('保存失败: '+((r&&r.error)||''),'error'); + }); +} +function runAgent(){ + const prompt=document.getElementById('agent-prompt').value.trim(); + if(!prompt){showToast('请输入指令','error');return;} + apiPost('/api/agent/run',{prompt}).then(r=>{ + if(r&&r.ok){ + showToast('Agent 已启动','success'); + document.getElementById('agent-status').textContent='运行中...'; + document.getElementById('btn-agent-run').disabled=true; + document.getElementById('agent-answer-wrap').style.display='none'; + startAgentPoll(); + }else showToast('启动失败: '+((r&&r.error)||''),'error'); + }); +} +function startAgentPoll(){ + if(_agentPoll)clearInterval(_agentPoll); + _agentPoll=setInterval(pollAgent,2000); + pollAgent(); +} +function stopAgentPoll(){ + if(_agentPoll){clearInterval(_agentPoll);_agentPoll=null;} +} +function pollAgent(){ + apiGet('/api/agent/status').then(r=>{ + if(!r||!r.ok)return; + const st=document.getElementById('agent-status'); + if(r.state==='running'){ + st.textContent='运行中... ('+(r.steps||[]).length+' 步)'; + }else{ + st.textContent=r.state==='done'?'完成':'失败'; + document.getElementById('btn-agent-run').disabled=false; + stopAgentPoll(); + } + // 步骤流（含截图缩略） + const steps=document.getElementById('agent-steps'); + const html=(r.steps||[]).map(s=>{ + const args=esc(s.args||''); + const img=s.image + ?'':''; + return '
' + +''+esc(s.tool||'')+' ' + +''+args+''+img+'
'; + }).join('')||''; + steps.innerHTML=html; + // 最终回答 + if(r.answer){ + const wrap=document.getElementById('agent-answer-wrap'); + wrap.style.display='block'; + document.getElementById('agent-answer').textContent=r.answer; + } + if(r.error){ + st.textContent='错误: '+r.error; + document.getElementById('btn-agent-run').disabled=false; + stopAgentPoll(); + } + }); +} + // ================== 设备自动发现（扫描 → 待连接池 → 确认） ================== function loadDiscovery(){ apiGet('/api/devices/discovery').then(r=>{ diff --git a/templates/admin/monitor.html b/templates/admin/monitor.html index 9f3f432..52480c7 100644 --- a/templates/admin/monitor.html +++ b/templates/admin/monitor.html @@ -480,6 +480,7 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14 + @@ -702,6 +703,37 @@ body{background:var(--bg);font-family:var(--body);color:var(--text);font-size:14
管理设备分组，用于任务按分组执行。分组内的设备需先在设备池管理中确认（不在池内的设备不参与调度）。
+ +
+
+ ⚙ 模型配置（OpenAI 兼容 API） +
+ + + + + + +
+
+
+ + + +
+
+ +
AI 通过截图观察屏幕并执行操作（元素定位优先，坐标兜底）。模型与 Key 配置保存在平台（管理员可见），请勿填他人密钥。
+
+ diff --git a/web/__init__.py b/web/__init__.py index dfd8ee3..81696b9 100644 --- a/web/__init__.py +++ b/web/__init__.py @@ -23,6 +23,7 @@ def register_blueprints(app): from .devices_api import bp as devices_bp from .apks_api import bp as apks_bp from .tailscale_api import bp as tailscale_bp + from .agent_api import bp as agent_bp for bp in (auth_bp, monitor_bp, tasks_bp, admin_bp, tools_bp, - devices_bp, apks_bp, tailscale_bp): + devices_bp, apks_bp, tailscale_bp, agent_bp): app.register_blueprint(bp) diff --git a/web/agent_api.py b/web/agent_api.py new file mode 100644 index 0000000..111d333 --- /dev/null +++ b/web/agent_api.py @@ -0,0 +1,157 @@ +"""AI 控制台 API：Agent 配置（模型/Key 前端可配）+ 运行 + 状态轮询。 + +Agent 在平台进程内跑（后台线程），连本机 MCP Server（8033）执行工具。 +单实例：同一时间只允许一个 Agent 运行，避免多路操作设备冲突。 +""" +import asyncio +import base64 +import io +import os +import sys +import threading +import uuid + +from flask import Blueprint, jsonify, request + +from core.logger import get_logger +from core.models import db +from web.auth import admin_required + +_log = get_logger("web.agent") +bp = Blueprint("agent", __name__) + +# ---------- 配置键（app_meta） ---------- +_CFG_KEYS = {"api_base": "agent_api_base", + "model": "agent_model", + "api_key": "agent_api_key", + "default_serial": "agent_default_serial"} + +# ---------- 运行状态（单实例） ---------- +_run = {"id": None, "state": "idle", "prompt": "", "serial": "", + "steps": [], "answer": "", "error": ""} +_lock = threading.Lock() + + +def _meta_get(key): + return db.session.execute( + db.text("SELECT value FROM app_meta WHERE key=:k"), {"k": key}).scalar() or "" + + +def _meta_put(key, value): + db.session.execute( + db.text("INSERT OR REPLACE INTO app_meta(key,value) VALUES(:k,:v)"), + {"k": key, "v": str(value)}) + + +def _read_cfg(): + return {k: _meta_get(v) for k, v in _CFG_KEYS.items()} + + +@bp.route("/api/agent/config", methods=["GET"]) +@admin_required +def agent_config_get(): + """读 Agent 配置（key 打码返回）。""" + cfg = _read_cfg() + if cfg["api_key"]: + k = cfg["api_key"] + cfg["api_key_masked"] = k[:6] + "***" + k[-4:] + return jsonify({"ok": True, **cfg}) + + +@bp.route("/api/agent/config", methods=["POST"]) +@admin_required +def agent_config_save(): + """保存 Agent 配置：{api_base?, model?, api_key?, default_serial?} 部分更新。""" + data = request.json or {} + for key, meta_key in _CFG_KEYS.items(): + if key in data and data[key] is not None: + _meta_put(meta_key, str(data[key]).strip()) + db.session.commit() + return jsonify({"ok": True, "msg": "已保存"}) + + +@bp.route("/api/agent/run", methods=["POST"]) +@admin_required +def agent_run(): + """启动 Agent 执行指令：{prompt, serial?}。运行中返回 409。""" + data = request.json or {} + prompt = (data.get("prompt") or "").strip() + if not prompt: + return jsonify({"ok": False, "error": "请输入指令"}), 400 + cfg = _read_cfg() + if not cfg.get("api_key"): + return jsonify({"ok": False, "error": "请先在配置区填写 API Key"}), 400 + with _lock: + if _run["state"] == "running": + return jsonify({"ok": False, "error": "已有 Agent 运行中，请等待完成"}), 409 + run_id = uuid.uuid4().hex[:8] + _run.update(id=run_id, state="running", prompt=prompt, + serial=(data.get("serial") or "").strip(), + steps=[], answer="", error="") + _log.info(f"Agent 启动: {prompt[:60]} serial={_run['serial']}") + threading.Thread(target=_agent_thread, args=(run_id, prompt, cfg), + daemon=True).start() + return jsonify({"ok": True, "run_id": run_id}) + + +@bp.route("/api/agent/status") +@admin_required +def agent_status(): + """Agent 运行状态（前端轮询）：{state, prompt, serial, steps, answer, error}。""" + with _lock: + return jsonify({"ok": True, **_run}) + + +def _shrink_image(b64, width=220, quality=50): + """截图降采样（web 展示用，避免大图撑爆轮询响应）。失败原样返回。""" + try: + from PIL import Image + img = Image.open(io.BytesIO(base64.b64decode(b64))) + if img.width > width: + img = img.resize((width, int(img.height * width / img.width))) + buf = io.BytesIO() + img.convert("RGB").save(buf, "JPEG", quality=quality) + return base64.b64encode(buf.getvalue()).decode() + except Exception: + return b64 + + +def _agent_thread(run_id, prompt, cfg): + """后台线程：跑 Agent，回调记录步骤。""" + try: + # 平台进程 cwd=/app（含 mcp_agent 包），进程内 import + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + from mcp_agent.agent import Agent + + def on_step(step): + with _lock: + if _run["id"] != run_id: + return + rec = {"tool": step.get("tool"), "args": str(step.get("args"))[:120]} + if step.get("image_b64"): + rec["image"] = _shrink_image(step["image_b64"]) + # 保留最近 6 步截图，防止响应过大 + _run["steps"] = _run["steps"][-5:] + _run["steps"].append(rec) + + agent = Agent() + # 用 web 配置覆盖 Agent 默认（api key 等由前端配置） + agent.s.api_base = cfg.get("api_base") or agent.s.api_base + agent.s.model = cfg.get("model") or agent.s.model + agent.s.api_key = cfg.get("api_key") or agent.s.api_key + agent.s.default_serial = cfg.get("default_serial") or agent.s.default_serial + agent.on_step = on_step + + async def _run(): + await agent._load_tools() + return await agent.run(prompt, cfg.get("default_serial") or "") + + answer = asyncio.run(_run()) + with _lock: + _run["state"] = "done" + _run["answer"] = answer + except Exception as e: + _log.warning(f"Agent 运行异常: {e}") + with _lock: + _run["state"] = "error" + _run["error"] = f"{type(e).__name__}: {str(e)[:200]}"