Files
auto_control/mcp_server/mcp_server.py
T
butubb 79cdf61ed2 feat: 自动认领开关 + 全站按设备名称显示
一、指纹匹配自动认领(可选,默认关)
- 发现设置新增「指纹匹配自动认领」勾选(app_meta: discovery_auto_claim,默认 0)
- 打开后:扫描发现某设备指纹与池中已有记录一致(同一台换了 IP)→ 自动迁移记录到新地址
  并同步分组/任务引用,零点击;关闭时维持"识别自动 + 人工点一次确认"
- 默认关的原因:认领会改写分组/任务引用(数据结构变动),交人工确认更稳妥
- 扫描结果与状态行会显示本轮自动认领了几台

二、设备名称在界面上呈现(凡选择/展示设备处都显示名称)
- 新增前端 helper `devText(name, serial)`(base.js):有名称→「名称 · serial」
- 监控页设备表:名称加粗为主、地址作副行(未命名显示橙色提醒);任务概况的覆盖设备
  chip 也优先显示名称(tooltip 保留完整地址)
- AI 控制台:目标设备下拉、实时画面设备下拉、目标/运行中提示都带名称(serial→name 映射)
- 任务编辑器「指定设备」下拉、分组编辑的设备勾选列表:带名称
- 后端 `/api/devices` 新增 `items`([{serial,name,model}],`devices` 保持兼容);
  `/api/agent/devices` 增加 `name` 字段
- MCP `de_list_devices` 返回 `name`,并在工具说明与 Agent 系统提示里要求"汇报用名称、
  调工具用 serial"

文档:API.md(items/name/auto_claim + §6.2.1 名称呈现表)、MCP.md(工具返回)

自测(全通过):自动认领端到端(开开关→扫描→自动迁址 + 名称保留 + 分组/任务引用同步 +
待连接池清理 + 开关默认关且可持久化);名称显示浏览器验证(监控页/覆盖设备 chip/AI 目标与
观看下拉/任务编辑器/分组弹窗/发现设置开关);设备指纹与人工认领回归
2026-09-11 11:16:24 +08:00

524 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""MCP 手机控制 Server(M0:设备列表/截图/点击/滑动)。
运行:MCP_ALLOW_WRITE=1 python -m mcp_server.mcp_server
客户端:Streamable HTTP @ http://<host>:8033/mcp
"""
import base64
import io
import logging
from fastmcp import FastMCP
from PIL import Image
from mcp_server import audit, config
from mcp_server.platform_client import PlatformClient, PlatformError
logging.basicConfig(level=logging.INFO,
format="%(asctime)s %(levelname)s [%(name)s] %(message)s")
_log = logging.getLogger("mcp")
S = config.Settings()
# 坐标空间缓存:serial -> (display_w, display_h, native_w, native_h)
# de_screenshot 返回的图与 de_tap/de_swipe 的坐标同一空间(display),
# server 按比例换算为设备原生坐标(模型只感知截图坐标系,不感知原生分辨率)。
_coord = {}
audit.init(S.audit_file)
mcp = FastMCP("mobile-control")
_platform = None
def platform():
"""惰性初始化平台客户端(启动即登录,失败明确报错)。"""
global _platform
if _platform is None:
_platform = PlatformClient(S.platform_url, S.platform_user,
S.platform_pass, S.platform_timeout)
return _platform
def _check_serial(serial):
"""白名单校验:未配置时限制为平台设备池(enabled)设备。"""
if not serial:
raise PlatformError("invalid_param", "缺少 serial")
if S.allowed_serials and serial not in S.allowed_serials:
raise PlatformError("device_not_allowed", f"设备 {serial} 不在白名单")
return serial
def _check_write():
if not S.allow_write:
raise PlatformError("write_disabled", "写操作未启用(MCP_ALLOW_WRITE=1 开启)")
# 设备任务占用锁:serial -> (ts, worker_status, task_job)。AI 写操作前检查,
# 任务 running/connecting 的设备拒绝操作(AI 不与任务抢设备)。5s 缓存。
_busy_cache = {}
def _ensure_device_free(serial):
"""写操作前检查设备是否有任务在跑(worker running/connecting → 拒绝)。"""
import time
now = time.time()
c = _busy_cache.get(serial)
if not c or now - c[0] > 5:
try:
devs = platform().list_devices()
info = next((d for d in devs if d.get("serial") == serial), {})
c = (now, info.get("worker_status") or "idle",
info.get("task_job") or "")
_busy_cache[serial] = c
except Exception:
return # 状态查询失败不阻塞(操作失败会另行报错)
if c[1] in ("running", "connecting"):
raise PlatformError(
"device_busy",
f"设备正在执行任务「{c[2] or '未知'}」——AI 不与任务抢设备,"
f"任务结束后才能操作(可在平台任务页先停止任务)")
def _to_native(serial, x, y):
"""截图坐标 → 设备原生坐标(按最近一次截图的比例换算)。"""
c = _coord.get(serial)
if not c:
raise PlatformError("invalid_param",
"请先对该设备执行 de_screenshot(需要建立坐标空间)")
dw, dh, nw, nh = c
return (round(x * nw / dw), round(y * nh / dh))
def _err(e: PlatformError):
return {"ok": False, "error": {"code": e.code, "message": e.message}}
def _ok(data):
return {"ok": True, "data": data}
@mcp.tool()
def de_list_devices() -> dict:
"""列出可控制设备:名称/地址/在线状态/型号/任务状态/前台 App。
返回 [{serial, name, model, online, task_job, worker_status, foreground_app}]。
`name` 是设备在平台里的名称(设备池里的身份标识);对你的用户汇报时**优先用名称**
(多台同型号设备时靠它区分)。后续所有工具仍然用 `serial` 指定设备。
"""
try:
devs = platform().list_devices()
except PlatformError as e:
return _err(e)
out = []
for d in devs:
out.append({
"serial": d.get("serial"),
"name": d.get("device_name") or "",
"model": d.get("model") or "",
"online": bool(d.get("present")),
"task_job": d.get("task_job") or "",
"worker_status": d.get("worker_status") or "idle",
"foreground_app": d.get("foreground_app") or "",
})
audit.audit("de_list_devices", "", "", f"{len(out)} 台")
return _ok(out)
@mcp.tool()
def de_screenshot(serial: str) -> dict:
"""截取设备屏幕并返回图像(image/jpeg,宽 ≤540px)。
同时返回 {width, height, screen_state}。多模态客户端可直接看图。
"""
try:
serial = _check_serial(serial)
jpeg, screen_state = platform().screenshot(serial)
img = Image.open(io.BytesIO(jpeg))
w, h = img.size
if w > S.screenshot_width:
ratio = S.screenshot_width / w
img = img.resize((S.screenshot_width, int(h * ratio)))
buf = io.BytesIO()
img.convert("RGB").save(buf, "JPEG", quality=S.jpeg_quality)
data = base64.b64encode(buf.getvalue()).decode()
# 记录坐标空间(display=返回图尺寸,native=设备原生),供 tap/swipe 换算
nw, nh = platform().screen_size(serial)
_coord[serial] = (img.size[0], img.size[1], nw, nh)
except PlatformError as e:
return _err(e)
except Exception as e:
return _err(PlatformError("platform_unavailable", f"截图处理失败: {e}"))
audit.audit("de_screenshot", serial, f"{w}x{h}", "ok")
dw, dh, nw, nh = _coord[serial]
return _ok({"image": {"type": "image", "data": data,
"mimeType": "image/jpeg"},
"width": dw, "height": dh,
"native_size": {"width": nw, "height": nh},
"screen_state": screen_state})
@mcp.tool()
def de_tap(serial: str, x: int, y: int) -> dict:
"""点击设备屏幕指定坐标(坐标空间 = de_screenshot 的图像坐标)。
自动吸附:若该点落在某个可点击元素内,实际点击会改为该元素的中心——
坐标只需大致对准目标即可(模型视觉定位常有偏差,吸附保证点准);
点在空白处则按原坐标点击。返回中的 snapped/label 可核对吸附结果。
"""
try:
_check_write()
serial = _check_serial(serial)
_ensure_device_free(serial)
if x < 0 or y < 0:
raise PlatformError("invalid_param", "坐标不能为负")
nx, ny = _to_native(serial, x, y)
res = platform().tap(serial, nx, ny, snap=True)
except PlatformError as e:
return _err(e)
audit.audit("de_tap", serial,
f"({x},{y})->native({nx},{ny})"
+ (f" 吸附[{res.get('label')}]" if res.get("snapped") else ""),
"ok")
return _ok({"action": "tap", "serial": serial, "x": x, "y": y,
"snapped": bool(res.get("snapped")),
"label": res.get("label") or ""})
@mcp.tool()
def de_swipe(serial: str, x1: int, y1: int, x2: int, y2: int,
duration: float = 0.2) -> dict:
"""在设备屏幕上滑动(坐标空间同 de_tap:截图坐标,server 换算原生)。"""
try:
_check_write()
serial = _check_serial(serial)
_ensure_device_free(serial)
nx1, ny1 = _to_native(serial, x1, y1)
nx2, ny2 = _to_native(serial, x2, y2)
platform().swipe(serial, nx1, ny1, nx2, ny2, duration)
except PlatformError as e:
return _err(e)
audit.audit("de_swipe", serial, f"({x1},{y1})->({x2},{y2})", "ok")
return _ok({"action": "swipe", "serial": serial})
_KEYS = ("back", "home", "recent", "menu", "power", "volume_up",
"volume_down", "enter", "delete", "search", "camera")
@mcp.tool()
def de_ui_tree(serial: str, limit: int = 150) -> dict:
"""获取当前界面元素树(文本 JSON):每元素含 text/resource_id/description/class/bounds。
可点击元素排在前面(可点性优先)。多数场景不需要读整棵树——直接给
de_tap_text 一个屏幕上可见的文字即可自动定位点击;本工具用于确认界面
上有什么、元素文案是否与预想一致。limit 控制返回条数(默认 150,防 token 膨胀)。
"""
try:
serial = _check_serial(serial)
if limit < 1 or limit > 300:
raise PlatformError("invalid_param", "limit 需在 1-300 之间")
els = platform().ui_elements(serial)
except PlatformError as e:
return _err(e)
# 精简输出:去掉 suggested/深度噪音,保留可定位属性;可点击优先、有文案优先
slim = []
for e in els:
slim.append({
"text": e.get("text", "")[:50],
"id": e.get("resource_id", "")[:80],
"desc": e.get("description", "")[:50],
"class": e.get("class", "").split(".")[-1],
"clickable": e.get("clickable", "") == "true",
"bounds": e.get("bounds", ""),
})
slim.sort(key=lambda x: (not x["clickable"], not (x["text"] or x["desc"])))
audit.audit("de_ui_tree", serial, "", f"{len(slim)} 元素")
return _ok({"count": len(slim), "elements": slim[:limit]})
@mcp.tool()
def de_tap_element(serial: str, by: str, value: str, index: int = 1) -> dict:
"""按元素点击(不需要坐标):by=text|id|desc|text_contains|desc_contains。
text/id/desc 为精确匹配;text_contains/desc_contains 为子串模糊匹配
(只记得部分文字时用,如 by=text_contains value=搜索)。
元素驱动操作比坐标可靠(界面变化自适应);元素不存在时返回错误,
可改用 de_ui_tree 查元素 / de_tap_text 按屏幕文字点 / de_tap 坐标兜底。
index 用于多命中取第几个(默认 1)。
"""
try:
_check_write()
serial = _check_serial(serial)
_ensure_device_free(serial)
if by not in ("text", "id", "desc", "text_contains", "desc_contains"):
raise PlatformError("invalid_param",
"by 可选 text/id/desc/text_contains/desc_contains")
if not value or index < 1:
raise PlatformError("invalid_param", "value 不能为空且 index>=1")
import uiautomator2 as u2
d = u2.connect(serial)
kw = {"text": value} if by == "text" else (
{"resourceId": value} if by == "id" else (
{"description": value} if by == "desc" else (
{"textContains": value} if by == "text_contains"
else {"descriptionContains": value})))
if index > 1:
kw["instance"] = index - 1
el = d(**kw)
if not el.exists:
raise PlatformError("device_offline",
f"未找到元素({by}={value},index={index})——"
f"建议 de_ui_tree 查看实际元素或 de_tap 用坐标")
el.click()
except PlatformError as e:
return _err(e)
except Exception as e:
return _err(PlatformError("platform_unavailable",
f"元素点击失败: {type(e).__name__}: {str(e)[:100]}"))
audit.audit("de_tap_element", serial, f"{by}={value}#{index}", "ok")
return _ok({"action": "tap_element", "serial": serial,
"by": by, "value": value, "index": index})
@mcp.tool()
def de_read_clipboard(serial: str) -> dict:
"""读取设备当前剪贴板内容(ClipInject/atx-agent 通道读回,M1 起支持)。"""
try:
serial = _check_serial(serial)
import uiautomator2 as u2
d = u2.connect(serial)
text = d.clipboard
except PlatformError as e:
return _err(e)
except Exception as e:
return _err(PlatformError("platform_unavailable",
f"剪贴板读取失败: {type(e).__name__}: {str(e)[:100]}"))
audit.audit("de_read_clipboard", serial, "", f"{len(text or '')} 字符")
return _ok({"clipboard": text or ""})
@mcp.tool()
def de_wake(serial: str) -> dict:
"""点亮设备屏幕并解锁(熄屏时先调用它再截图)。"""
try:
_check_write()
serial = _check_serial(serial)
_ensure_device_free(serial)
platform().wake(serial)
except PlatformError as e:
return _err(e)
audit.audit("de_wake", serial, "", "ok")
return _ok({"action": "wake", "serial": serial})
@mcp.tool()
def de_press_key(serial: str, key: str) -> dict:
"""按设备按键:back/home/recent/menu/power/enter/delete 等。"""
try:
_check_write()
serial = _check_serial(serial)
_ensure_device_free(serial)
if key not in _KEYS:
raise PlatformError("invalid_param", f"不支持的按键: {key}(可选 {_KEYS})")
platform().press_key(serial, key)
except PlatformError as e:
return _err(e)
audit.audit("de_press_key", serial, key, "ok")
return _ok({"action": "press_key", "serial": serial, "key": key})
# ================== L2 扩展工具(轻量通道:adb/u2 直连,省 token) ==================
@mcp.tool()
def de_open_app(serial: str, package: str) -> dict:
"""打开 App(adb monkey 直启,最快路径)。package 为应用包名,如 com.ss.android.ugc.aweme。"""
try:
_check_write()
serial = _check_serial(serial)
_ensure_device_free(serial)
if not package:
raise PlatformError("invalid_param", "缺少包名")
from mcp_server import direct_ops
direct_ops.open_app(serial, package)
except PlatformError as e:
return _err(e)
except Exception as e:
return _err(PlatformError("platform_unavailable", f"打开失败: {str(e)[:100]}"))
audit.audit("de_open_app", serial, package, "ok")
return _ok({"action": "open_app", "package": package})
@mcp.tool()
def de_stop_app(serial: str, package: str) -> dict:
"""强制停止 App(am force-stop)。"""
try:
_check_write()
serial = _check_serial(serial)
_ensure_device_free(serial)
if not package:
raise PlatformError("invalid_param", "缺少包名")
from mcp_server import direct_ops
direct_ops.stop_app(serial, package)
except PlatformError as e:
return _err(e)
except Exception as e:
return _err(PlatformError("platform_unavailable", f"停止失败: {str(e)[:100]}"))
audit.audit("de_stop_app", serial, package, "ok")
return _ok({"action": "stop_app", "package": package})
@mcp.tool()
def de_foreground_app(serial: str) -> dict:
"""查询设备当前前台运行的 App 包名(轻量 dumpsys,不打扰设备)。"""
try:
serial = _check_serial(serial)
from mcp_server import direct_ops
pkg = direct_ops.foreground_app(serial)
except PlatformError as e:
return _err(e)
except Exception as e:
return _err(PlatformError("platform_unavailable", f"查询失败: {str(e)[:100]}"))
audit.audit("de_foreground_app", serial, "", pkg or "未知")
return _ok({"foreground_app": pkg or ""})
@mcp.tool()
def de_type_text(serial: str, text: str) -> dict:
"""向设备当前输入框输入文字(支持中文,直接 set_text 不依赖剪贴板)。"""
try:
_check_write()
serial = _check_serial(serial)
_ensure_device_free(serial)
if not text:
raise PlatformError("invalid_param", "内容为空")
from mcp_server import direct_ops
ok, msg = direct_ops.type_text(serial, text)
if not ok:
raise PlatformError("device_offline", msg)
except PlatformError as e:
return _err(e)
except Exception as e:
return _err(PlatformError("platform_unavailable", f"输入失败: {str(e)[:100]}"))
audit.audit("de_type_text", serial, f"{text[:30]}...", "ok")
return _ok({"action": "type_text"})
@mcp.tool()
def de_set_clipboard(serial: str, text: str) -> dict:
"""写入设备剪贴板(ClipInject 通道,读回验证)。"""
try:
_check_write()
serial = _check_serial(serial)
_ensure_device_free(serial)
if not text:
raise PlatformError("invalid_param", "内容为空")
from mcp_server import direct_ops
ok, msg = direct_ops.set_clipboard(serial, text)
if not ok:
raise PlatformError("device_offline", msg)
except PlatformError as e:
return _err(e)
except Exception as e:
return _err(PlatformError("platform_unavailable", f"注入失败: {str(e)[:100]}"))
audit.audit("de_set_clipboard", serial, f"{text[:30]}...", "ok")
return _ok({"action": "set_clipboard"})
@mcp.tool()
def de_sleep(serial: str) -> dict:
"""熄灭设备屏幕(运行中任务会中断,慎用)。"""
try:
_check_write()
serial = _check_serial(serial)
_ensure_device_free(serial)
platform().sleep(serial)
except PlatformError as e:
return _err(e)
audit.audit("de_sleep", serial, "", "ok")
return _ok({"action": "sleep"})
@mcp.tool()
def de_ocr(serial: str) -> dict:
"""OCR 识别当前屏幕文字(图片/画布/WebView 里 UI 树没有的文字也能识别)。
返回 [{text, score}]——搜屏幕关键词后可配合 de_tap_element/de_tap 操作。
"""
try:
serial = _check_serial(serial)
from mcp_server import direct_ops
results = direct_ops.ocr(serial)
except PlatformError as e:
return _err(e)
except Exception as e:
return _err(PlatformError("platform_unavailable", f"OCR 失败: {str(e)[:100]}"))
slim = [{"text": r["text"], "score": round(r["score"], 2)} for r in results]
audit.audit("de_ocr", serial, "", f"{len(slim)} 条")
return _ok({"count": len(slim), "texts": slim[:100]})
@mcp.tool()
def de_tap_text(serial: str, text: str) -> dict:
"""点击屏幕上显示该文字的位置(语义点击:一次调用完成「找到并点击」,无需坐标)。
想点带文字的按钮/列表项/标签/链接时用它:text 只需是屏幕上可见文字的
一部分(子串匹配,如「搜索」「立即购买」)。原生控件直接命中;
WebView/图片/画布里渲染的文字自动走 OCR 兜底。多命中点第一处(想点
更靠下的请把文字换独特些)。屏幕确实没有该文字时返回错误提示,
请截图确认后换关键词。比 de_tap 坐标点击可靠,涉及文字目标时优先使用。
"""
try:
_check_write()
serial = _check_serial(serial)
_ensure_device_free(serial)
if not text or len(text) > 100:
raise PlatformError("invalid_param", "text 不能为空且 ≤100 字符")
res = platform().tap_text(serial, text)
except PlatformError as e:
return _err(e)
if not res.get("found"):
err = PlatformError("text_not_found",
f"屏幕上未找到文字「{text}」——先 de_screenshot 看当前界面,"
f"换用屏幕上实际存在的文字;若文字在需滑动后才可见请先滑动")
audit.audit("de_tap_text", serial, f"「{text[:30]}」", "未找到")
return _err(err)
audit.audit("de_tap_text", serial,
f"「{text[:30]}」via {res.get('method')} @({res.get('x')},{res.get('y')})", "ok")
return _ok({"action": "tap_text", "serial": serial, "text": text,
"method": res.get("method"), "matched": res.get("matched") or text,
"x": res.get("x"), "y": res.get("y")})
@mcp.tool()
def de_list_apps(serial: str, keyword: str = "") -> dict:
"""列出设备第三方已装应用包名(可关键词过滤,如 keyword='douyin' 找抖音)。"""
try:
serial = _check_serial(serial)
from mcp_server import direct_ops
pkgs = direct_ops.list_apps(serial, keyword)
except PlatformError as e:
return _err(e)
except Exception as e:
return _err(PlatformError("platform_unavailable", f"查询失败: {str(e)[:100]}"))
audit.audit("de_list_apps", serial, keyword or "", f"{len(pkgs)} 个")
return _ok({"count": len(pkgs), "packages": pkgs[:200]})
@mcp.tool()
def de_list_tasks() -> dict:
"""列出平台任务计划(名称/启用状态/调度),供了解可自动化的任务。"""
try:
tasks = platform().list_tasks()
except PlatformError as e:
return _err(e)
audit.audit("de_list_tasks", "", "", f"{len(tasks)} 个")
return _ok({"tasks": tasks})
def main():
_log.info("MCP mobile-control 启动: allow_write=%s port=%s",
S.allow_write, S.http_port)
mcp.run(transport="http", host=S.http_host, port=S.http_port)
if __name__ == "__main__":
main()