"""MCP 手机控制 Server(M0:设备列表/截图/点击/滑动)。 运行:MCP_ALLOW_WRITE=1 python -m mcp_server.mcp_server 客户端:Streamable HTTP @ http://:8033/mcp """ import base64 import io import logging from fastmcp import FastMCP from PIL import Image from mcp_server import audit, config from mcp_server.platform_client import PlatformClient, PlatformError logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s [%(name)s] %(message)s") _log = logging.getLogger("mcp") S = config.Settings() # 坐标空间缓存:serial -> (display_w, display_h, native_w, native_h) # de_screenshot 返回的图与 de_tap/de_swipe 的坐标同一空间(display), # server 按比例换算为设备原生坐标(模型只感知截图坐标系,不感知原生分辨率)。 _coord = {} audit.init(S.audit_file) mcp = FastMCP("mobile-control") _platform = None def platform(): """惰性初始化平台客户端(启动即登录,失败明确报错)。""" global _platform if _platform is None: _platform = PlatformClient(S.platform_url, S.platform_user, S.platform_pass, S.platform_timeout) return _platform def _check_serial(serial): """白名单校验:未配置时限制为平台设备池(enabled)设备。""" if not serial: raise PlatformError("invalid_param", "缺少 serial") if S.allowed_serials and serial not in S.allowed_serials: raise PlatformError("device_not_allowed", f"设备 {serial} 不在白名单") return serial def _check_write(): if not S.allow_write: raise PlatformError("write_disabled", "写操作未启用(MCP_ALLOW_WRITE=1 开启)") def _to_native(serial, x, y): """截图坐标 → 设备原生坐标(按最近一次截图的比例换算)。""" c = _coord.get(serial) if not c: raise PlatformError("invalid_param", "请先对该设备执行 de_screenshot(需要建立坐标空间)") dw, dh, nw, nh = c return (round(x * nw / dw), round(y * nh / dh)) def _err(e: PlatformError): return {"ok": False, "error": {"code": e.code, "message": e.message}} def _ok(data): return {"ok": True, "data": data} @mcp.tool() def de_list_devices() -> dict: """列出可控制设备:serial/在线状态/型号/任务状态/前台 App。 返回 [{serial, model, online, task_job, worker_status, foreground_app}]。 """ try: devs = platform().list_devices() except PlatformError as e: return _err(e) out = [] for d in devs: out.append({ "serial": d.get("serial"), "model": d.get("model") or d.get("device_name") or "", "online": bool(d.get("present")), "task_job": d.get("task_job") or "", "worker_status": d.get("worker_status") or "idle", "foreground_app": d.get("foreground_app") or "", }) audit.audit("de_list_devices", "", "", f"{len(out)} 台") return _ok(out) @mcp.tool() def de_screenshot(serial: str) -> dict: """截取设备屏幕并返回图像(image/jpeg,宽 ≤540px)。 同时返回 {width, height, screen_state}。多模态客户端可直接看图。 """ try: serial = _check_serial(serial) jpeg, screen_state = platform().screenshot(serial) img = Image.open(io.BytesIO(jpeg)) w, h = img.size if w > S.screenshot_width: ratio = S.screenshot_width / w img = img.resize((S.screenshot_width, int(h * ratio))) buf = io.BytesIO() img.convert("RGB").save(buf, "JPEG", quality=S.jpeg_quality) data = base64.b64encode(buf.getvalue()).decode() # 记录坐标空间(display=返回图尺寸,native=设备原生),供 tap/swipe 换算 nw, nh = platform().screen_size(serial) _coord[serial] = (img.size[0], img.size[1], nw, nh) except PlatformError as e: return _err(e) except Exception as e: return _err(PlatformError("platform_unavailable", f"截图处理失败: {e}")) audit.audit("de_screenshot", serial, f"{w}x{h}", "ok") dw, dh, nw, nh = _coord[serial] return _ok({"image": {"type": "image", "data": data, "mimeType": "image/jpeg"}, "width": dw, "height": dh, "native_size": {"width": nw, "height": nh}, "screen_state": screen_state}) @mcp.tool() def de_tap(serial: str, x: int, y: int) -> dict: """点击设备屏幕指定坐标(坐标空间 = de_screenshot 的图像坐标)。 自动吸附:若该点落在某个可点击元素内,实际点击会改为该元素的中心—— 坐标只需大致对准目标即可(模型视觉定位常有偏差,吸附保证点准); 点在空白处则按原坐标点击。返回中的 snapped/label 可核对吸附结果。 """ try: _check_write() serial = _check_serial(serial) if x < 0 or y < 0: raise PlatformError("invalid_param", "坐标不能为负") nx, ny = _to_native(serial, x, y) res = platform().tap(serial, nx, ny, snap=True) except PlatformError as e: return _err(e) audit.audit("de_tap", serial, f"({x},{y})->native({nx},{ny})" + (f" 吸附[{res.get('label')}]" if res.get("snapped") else ""), "ok") return _ok({"action": "tap", "serial": serial, "x": x, "y": y, "snapped": bool(res.get("snapped")), "label": res.get("label") or ""}) @mcp.tool() def de_swipe(serial: str, x1: int, y1: int, x2: int, y2: int, duration: float = 0.2) -> dict: """在设备屏幕上滑动(坐标空间同 de_tap:截图坐标,server 换算原生)。""" try: _check_write() serial = _check_serial(serial) nx1, ny1 = _to_native(serial, x1, y1) nx2, ny2 = _to_native(serial, x2, y2) platform().swipe(serial, nx1, ny1, nx2, ny2, duration) except PlatformError as e: return _err(e) audit.audit("de_swipe", serial, f"({x1},{y1})->({x2},{y2})", "ok") return _ok({"action": "swipe", "serial": serial}) _KEYS = ("back", "home", "recent", "menu", "power", "volume_up", "volume_down", "enter", "delete", "search", "camera") @mcp.tool() def de_ui_tree(serial: str, limit: int = 150) -> dict: """获取当前界面元素树(文本 JSON):每元素含 text/resource_id/description/class/bounds。 可点击元素排在前面(可点性优先)。多数场景不需要读整棵树——直接给 de_tap_text 一个屏幕上可见的文字即可自动定位点击;本工具用于确认界面 上有什么、元素文案是否与预想一致。limit 控制返回条数(默认 150,防 token 膨胀)。 """ try: serial = _check_serial(serial) if limit < 1 or limit > 300: raise PlatformError("invalid_param", "limit 需在 1-300 之间") els = platform().ui_elements(serial) except PlatformError as e: return _err(e) # 精简输出:去掉 suggested/深度噪音,保留可定位属性;可点击优先、有文案优先 slim = [] for e in els: slim.append({ "text": e.get("text", "")[:50], "id": e.get("resource_id", "")[:80], "desc": e.get("description", "")[:50], "class": e.get("class", "").split(".")[-1], "clickable": e.get("clickable", "") == "true", "bounds": e.get("bounds", ""), }) slim.sort(key=lambda x: (not x["clickable"], not (x["text"] or x["desc"]))) audit.audit("de_ui_tree", serial, "", f"{len(slim)} 元素") return _ok({"count": len(slim), "elements": slim[:limit]}) @mcp.tool() def de_tap_element(serial: str, by: str, value: str, index: int = 1) -> dict: """按元素点击(不需要坐标):by=text|id|desc|text_contains|desc_contains。 text/id/desc 为精确匹配;text_contains/desc_contains 为子串模糊匹配 (只记得部分文字时用,如 by=text_contains value=搜索)。 元素驱动操作比坐标可靠(界面变化自适应);元素不存在时返回错误, 可改用 de_ui_tree 查元素 / de_tap_text 按屏幕文字点 / de_tap 坐标兜底。 index 用于多命中取第几个(默认 1)。 """ try: _check_write() serial = _check_serial(serial) if by not in ("text", "id", "desc", "text_contains", "desc_contains"): raise PlatformError("invalid_param", "by 可选 text/id/desc/text_contains/desc_contains") if not value or index < 1: raise PlatformError("invalid_param", "value 不能为空且 index>=1") import uiautomator2 as u2 d = u2.connect(serial) kw = {"text": value} if by == "text" else ( {"resourceId": value} if by == "id" else ( {"description": value} if by == "desc" else ( {"textContains": value} if by == "text_contains" else {"descriptionContains": value}))) if index > 1: kw["instance"] = index - 1 el = d(**kw) if not el.exists: raise PlatformError("device_offline", f"未找到元素({by}={value},index={index})——" f"建议 de_ui_tree 查看实际元素或 de_tap 用坐标") el.click() except PlatformError as e: return _err(e) except Exception as e: return _err(PlatformError("platform_unavailable", f"元素点击失败: {type(e).__name__}: {str(e)[:100]}")) audit.audit("de_tap_element", serial, f"{by}={value}#{index}", "ok") return _ok({"action": "tap_element", "serial": serial, "by": by, "value": value, "index": index}) @mcp.tool() def de_read_clipboard(serial: str) -> dict: """读取设备当前剪贴板内容(ClipInject/atx-agent 通道读回,M1 起支持)。""" try: serial = _check_serial(serial) import uiautomator2 as u2 d = u2.connect(serial) text = d.clipboard except PlatformError as e: return _err(e) except Exception as e: return _err(PlatformError("platform_unavailable", f"剪贴板读取失败: {type(e).__name__}: {str(e)[:100]}")) audit.audit("de_read_clipboard", serial, "", f"{len(text or '')} 字符") return _ok({"clipboard": text or ""}) @mcp.tool() def de_wake(serial: str) -> dict: """点亮设备屏幕并解锁(熄屏时先调用它再截图)。""" try: _check_write() serial = _check_serial(serial) platform().wake(serial) except PlatformError as e: return _err(e) audit.audit("de_wake", serial, "", "ok") return _ok({"action": "wake", "serial": serial}) @mcp.tool() def de_press_key(serial: str, key: str) -> dict: """按设备按键:back/home/recent/menu/power/enter/delete 等。""" try: _check_write() serial = _check_serial(serial) if key not in _KEYS: raise PlatformError("invalid_param", f"不支持的按键: {key}(可选 {_KEYS})") platform().press_key(serial, key) except PlatformError as e: return _err(e) audit.audit("de_press_key", serial, key, "ok") return _ok({"action": "press_key", "serial": serial, "key": key}) # ================== L2 扩展工具(轻量通道:adb/u2 直连,省 token) ================== @mcp.tool() def de_open_app(serial: str, package: str) -> dict: """打开 App(adb monkey 直启,最快路径)。package 为应用包名,如 com.ss.android.ugc.aweme。""" try: _check_write() serial = _check_serial(serial) if not package: raise PlatformError("invalid_param", "缺少包名") from mcp_server import direct_ops direct_ops.open_app(serial, package) except PlatformError as e: return _err(e) except Exception as e: return _err(PlatformError("platform_unavailable", f"打开失败: {str(e)[:100]}")) audit.audit("de_open_app", serial, package, "ok") return _ok({"action": "open_app", "package": package}) @mcp.tool() def de_stop_app(serial: str, package: str) -> dict: """强制停止 App(am force-stop)。""" try: _check_write() serial = _check_serial(serial) if not package: raise PlatformError("invalid_param", "缺少包名") from mcp_server import direct_ops direct_ops.stop_app(serial, package) except PlatformError as e: return _err(e) except Exception as e: return _err(PlatformError("platform_unavailable", f"停止失败: {str(e)[:100]}")) audit.audit("de_stop_app", serial, package, "ok") return _ok({"action": "stop_app", "package": package}) @mcp.tool() def de_foreground_app(serial: str) -> dict: """查询设备当前前台运行的 App 包名(轻量 dumpsys,不打扰设备)。""" try: serial = _check_serial(serial) from mcp_server import direct_ops pkg = direct_ops.foreground_app(serial) except PlatformError as e: return _err(e) except Exception as e: return _err(PlatformError("platform_unavailable", f"查询失败: {str(e)[:100]}")) audit.audit("de_foreground_app", serial, "", pkg or "未知") return _ok({"foreground_app": pkg or ""}) @mcp.tool() def de_type_text(serial: str, text: str) -> dict: """向设备当前输入框输入文字(支持中文,直接 set_text 不依赖剪贴板)。""" try: _check_write() serial = _check_serial(serial) if not text: raise PlatformError("invalid_param", "内容为空") from mcp_server import direct_ops ok, msg = direct_ops.type_text(serial, text) if not ok: raise PlatformError("device_offline", msg) except PlatformError as e: return _err(e) except Exception as e: return _err(PlatformError("platform_unavailable", f"输入失败: {str(e)[:100]}")) audit.audit("de_type_text", serial, f"{text[:30]}...", "ok") return _ok({"action": "type_text"}) @mcp.tool() def de_set_clipboard(serial: str, text: str) -> dict: """写入设备剪贴板(ClipInject 通道,读回验证)。""" try: _check_write() serial = _check_serial(serial) if not text: raise PlatformError("invalid_param", "内容为空") from mcp_server import direct_ops ok, msg = direct_ops.set_clipboard(serial, text) if not ok: raise PlatformError("device_offline", msg) except PlatformError as e: return _err(e) except Exception as e: return _err(PlatformError("platform_unavailable", f"注入失败: {str(e)[:100]}")) audit.audit("de_set_clipboard", serial, f"{text[:30]}...", "ok") return _ok({"action": "set_clipboard"}) @mcp.tool() def de_sleep(serial: str) -> dict: """熄灭设备屏幕(运行中任务会中断,慎用)。""" try: _check_write() serial = _check_serial(serial) platform().sleep(serial) except PlatformError as e: return _err(e) audit.audit("de_sleep", serial, "", "ok") return _ok({"action": "sleep"}) @mcp.tool() def de_ocr(serial: str) -> dict: """OCR 识别当前屏幕文字(图片/画布/WebView 里 UI 树没有的文字也能识别)。 返回 [{text, score}]——搜屏幕关键词后可配合 de_tap_element/de_tap 操作。 """ try: serial = _check_serial(serial) from mcp_server import direct_ops results = direct_ops.ocr(serial) except PlatformError as e: return _err(e) except Exception as e: return _err(PlatformError("platform_unavailable", f"OCR 失败: {str(e)[:100]}")) slim = [{"text": r["text"], "score": round(r["score"], 2)} for r in results] audit.audit("de_ocr", serial, "", f"{len(slim)} 条") return _ok({"count": len(slim), "texts": slim[:100]}) @mcp.tool() def de_tap_text(serial: str, text: str) -> dict: """点击屏幕上显示该文字的位置(语义点击:一次调用完成「找到并点击」,无需坐标)。 想点带文字的按钮/列表项/标签/链接时用它:text 只需是屏幕上可见文字的 一部分(子串匹配,如「搜索」「立即购买」)。原生控件直接命中; WebView/图片/画布里渲染的文字自动走 OCR 兜底。多命中点第一处(想点 更靠下的请把文字换独特些)。屏幕确实没有该文字时返回错误提示, 请截图确认后换关键词。比 de_tap 坐标点击可靠,涉及文字目标时优先使用。 """ try: _check_write() serial = _check_serial(serial) if not text or len(text) > 100: raise PlatformError("invalid_param", "text 不能为空且 ≤100 字符") res = platform().tap_text(serial, text) except PlatformError as e: return _err(e) if not res.get("found"): err = PlatformError("text_not_found", f"屏幕上未找到文字「{text}」——先 de_screenshot 看当前界面," f"换用屏幕上实际存在的文字;若文字在需滑动后才可见请先滑动") audit.audit("de_tap_text", serial, f"「{text[:30]}」", "未找到") return _err(err) audit.audit("de_tap_text", serial, f"「{text[:30]}」via {res.get('method')} @({res.get('x')},{res.get('y')})", "ok") return _ok({"action": "tap_text", "serial": serial, "text": text, "method": res.get("method"), "matched": res.get("matched") or text, "x": res.get("x"), "y": res.get("y")}) @mcp.tool() def de_list_apps(serial: str, keyword: str = "") -> dict: """列出设备第三方已装应用包名(可关键词过滤,如 keyword='douyin' 找抖音)。""" try: serial = _check_serial(serial) from mcp_server import direct_ops pkgs = direct_ops.list_apps(serial, keyword) except PlatformError as e: return _err(e) except Exception as e: return _err(PlatformError("platform_unavailable", f"查询失败: {str(e)[:100]}")) audit.audit("de_list_apps", serial, keyword or "", f"{len(pkgs)} 个") return _ok({"count": len(pkgs), "packages": pkgs[:200]}) @mcp.tool() def de_list_tasks() -> dict: """列出平台任务计划(名称/启用状态/调度),供了解可自动化的任务。""" try: tasks = platform().list_tasks() except PlatformError as e: return _err(e) audit.audit("de_list_tasks", "", "", f"{len(tasks)} 个") return _ok({"tasks": tasks}) def main(): _log.info("MCP mobile-control 启动: allow_write=%s port=%s", S.allow_write, S.http_port) mcp.run(transport="http", host=S.http_host, port=S.http_port) if __name__ == "__main__": main()