"""MCP 手机控制 Server(M0:设备列表/截图/点击/滑动)。 运行:MCP_ALLOW_WRITE=1 python -m mcp_server.mcp_server 客户端:Streamable HTTP @ http://:8033/mcp """ import base64 import io import logging from fastmcp import FastMCP from PIL import Image from mcp_server import audit, config from mcp_server.platform_client import PlatformClient, PlatformError logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s [%(name)s] %(message)s") _log = logging.getLogger("mcp") S = config.Settings() # 坐标空间缓存:serial -> (display_w, display_h, native_w, native_h) # de_screenshot 返回的图与 de_tap/de_swipe 的坐标同一空间(display), # server 按比例换算为设备原生坐标(模型只感知截图坐标系,不感知原生分辨率)。 _coord = {} audit.init(S.audit_file) mcp = FastMCP("mobile-control") _platform = None def platform(): """惰性初始化平台客户端(启动即登录,失败明确报错)。""" global _platform if _platform is None: _platform = PlatformClient(S.platform_url, S.platform_user, S.platform_pass, S.platform_timeout) return _platform def _check_serial(serial): """白名单校验:未配置时限制为平台设备池(enabled)设备。""" if not serial: raise PlatformError("invalid_param", "缺少 serial") if S.allowed_serials and serial not in S.allowed_serials: raise PlatformError("device_not_allowed", f"设备 {serial} 不在白名单") return serial def _check_write(): if not S.allow_write: raise PlatformError("write_disabled", "写操作未启用(MCP_ALLOW_WRITE=1 开启)") def _to_native(serial, x, y): """截图坐标 → 设备原生坐标(按最近一次截图的比例换算)。""" c = _coord.get(serial) if not c: raise PlatformError("invalid_param", "请先对该设备执行 de_screenshot(需要建立坐标空间)") dw, dh, nw, nh = c return (round(x * nw / dw), round(y * nh / dh)) def _err(e: PlatformError): return {"ok": False, "error": {"code": e.code, "message": e.message}} def _ok(data): return {"ok": True, "data": data} @mcp.tool() def de_list_devices() -> dict: """列出可控制设备:serial/在线状态/型号/任务状态/前台 App。 返回 [{serial, model, online, task_job, worker_status, foreground_app}]。 """ try: devs = platform().list_devices() except PlatformError as e: return _err(e) out = [] for d in devs: out.append({ "serial": d.get("serial"), "model": d.get("model") or d.get("device_name") or "", "online": bool(d.get("present")), "task_job": d.get("task_job") or "", "worker_status": d.get("worker_status") or "idle", "foreground_app": d.get("foreground_app") or "", }) audit.audit("de_list_devices", "", "", f"{len(out)} 台") return _ok(out) @mcp.tool() def de_screenshot(serial: str) -> dict: """截取设备屏幕并返回图像(image/jpeg,宽 ≤540px)。 同时返回 {width, height, screen_state}。多模态客户端可直接看图。 """ try: serial = _check_serial(serial) jpeg, screen_state = platform().screenshot(serial) img = Image.open(io.BytesIO(jpeg)) w, h = img.size if w > S.screenshot_width: ratio = S.screenshot_width / w img = img.resize((S.screenshot_width, int(h * ratio))) buf = io.BytesIO() img.convert("RGB").save(buf, "JPEG", quality=S.jpeg_quality) data = base64.b64encode(buf.getvalue()).decode() # 记录坐标空间(display=返回图尺寸,native=设备原生),供 tap/swipe 换算 nw, nh = platform().screen_size(serial) _coord[serial] = (img.size[0], img.size[1], nw, nh) except PlatformError as e: return _err(e) except Exception as e: return _err(PlatformError("platform_unavailable", f"截图处理失败: {e}")) audit.audit("de_screenshot", serial, f"{w}x{h}", "ok") dw, dh, nw, nh = _coord[serial] return _ok({"image": {"type": "image", "data": data, "mimeType": "image/jpeg"}, "width": dw, "height": dh, "native_size": {"width": nw, "height": nh}, "screen_state": screen_state}) @mcp.tool() def de_tap(serial: str, x: int, y: int) -> dict: """点击设备屏幕指定坐标。 坐标空间 = de_screenshot 返回的图像坐标(display 空间)——先截图拿到 native_size 后再点击,server 自动换算为设备原生坐标。 """ try: _check_write() serial = _check_serial(serial) if x < 0 or y < 0: raise PlatformError("invalid_param", "坐标不能为负") nx, ny = _to_native(serial, x, y) platform().tap(serial, nx, ny) except PlatformError as e: return _err(e) audit.audit("de_tap", serial, f"({x},{y})->native({nx},{ny})", "ok") return _ok({"action": "tap", "serial": serial, "x": x, "y": y}) @mcp.tool() def de_swipe(serial: str, x1: int, y1: int, x2: int, y2: int, duration: float = 0.2) -> dict: """在设备屏幕上滑动(坐标空间同 de_tap:截图坐标,server 换算原生)。""" try: _check_write() serial = _check_serial(serial) nx1, ny1 = _to_native(serial, x1, y1) nx2, ny2 = _to_native(serial, x2, y2) platform().swipe(serial, nx1, ny1, nx2, ny2, duration) except PlatformError as e: return _err(e) audit.audit("de_swipe", serial, f"({x1},{y1})->({x2},{y2})", "ok") return _ok({"action": "swipe", "serial": serial}) _KEYS = ("back", "home", "recent", "menu", "power", "volume_up", "volume_down", "enter", "delete", "search", "camera") @mcp.tool() def de_ui_tree(serial: str) -> dict: """获取当前界面元素树(文本 JSON):每元素含 text/resource_id/description/class/bounds。 优先用它定位元素(元素驱动操作),比纯坐标点击更可靠。 """ try: serial = _check_serial(serial) els = platform().ui_elements(serial) except PlatformError as e: return _err(e) # 精简输出:去掉 suggested/深度噪音,保留可定位属性 slim = [] for e in els: slim.append({ "text": e.get("text", "")[:50], "id": e.get("resource_id", "")[:80], "desc": e.get("description", "")[:50], "class": e.get("class", "").split(".")[-1], "bounds": e.get("bounds", ""), }) audit.audit("de_ui_tree", serial, "", f"{len(slim)} 元素") return _ok({"count": len(slim), "elements": slim[:300]}) @mcp.tool() def de_tap_element(serial: str, by: str, value: str, index: int = 1) -> dict: """按元素点击(不需要坐标):by=text|id|desc,value 为匹配文本/资源 id/描述。 元素驱动操作比坐标可靠(界面变化自适应);元素不存在时返回错误, 可改用 de_ui_tree 查元素或 de_tap 坐标兜底。index 用于多命中取第几个(默认 1)。 """ try: _check_write() serial = _check_serial(serial) if by not in ("text", "id", "desc"): raise PlatformError("invalid_param", "by 可选 text/id/desc") if not value or index < 1: raise PlatformError("invalid_param", "value 不能为空且 index>=1") import uiautomator2 as u2 d = u2.connect(serial) kw = {"text": value} if by == "text" else ( {"resourceId": value} if by == "id" else {"description": value}) if index > 1: kw["instance"] = index - 1 el = d(**kw) if not el.exists: raise PlatformError("device_offline", f"未找到元素({by}={value},index={index})——" f"建议 de_ui_tree 查看实际元素或 de_tap 用坐标") el.click() except PlatformError as e: return _err(e) except Exception as e: return _err(PlatformError("platform_unavailable", f"元素点击失败: {type(e).__name__}: {str(e)[:100]}")) audit.audit("de_tap_element", serial, f"{by}={value}#{index}", "ok") return _ok({"action": "tap_element", "serial": serial, "by": by, "value": value, "index": index}) @mcp.tool() def de_read_clipboard(serial: str) -> dict: """读取设备当前剪贴板内容(ClipInject/atx-agent 通道读回,M1 起支持)。""" try: serial = _check_serial(serial) import uiautomator2 as u2 d = u2.connect(serial) text = d.clipboard except PlatformError as e: return _err(e) except Exception as e: return _err(PlatformError("platform_unavailable", f"剪贴板读取失败: {type(e).__name__}: {str(e)[:100]}")) audit.audit("de_read_clipboard", serial, "", f"{len(text or '')} 字符") return _ok({"clipboard": text or ""}) @mcp.tool() def de_wake(serial: str) -> dict: """点亮设备屏幕并解锁(熄屏时先调用它再截图)。""" try: _check_write() serial = _check_serial(serial) platform().wake(serial) except PlatformError as e: return _err(e) audit.audit("de_wake", serial, "", "ok") return _ok({"action": "wake", "serial": serial}) @mcp.tool() def de_press_key(serial: str, key: str) -> dict: """按设备按键:back/home/recent/menu/power/enter/delete 等。""" try: _check_write() serial = _check_serial(serial) if key not in _KEYS: raise PlatformError("invalid_param", f"不支持的按键: {key}(可选 {_KEYS})") platform().press_key(serial, key) except PlatformError as e: return _err(e) audit.audit("de_press_key", serial, key, "ok") return _ok({"action": "press_key", "serial": serial, "key": key}) def main(): _log.info("MCP mobile-control 启动: allow_write=%s port=%s", S.allow_write, S.http_port) mcp.run(transport="http", host=S.http_host, port=S.http_port) if __name__ == "__main__": main()