267 lines
9.6 KiB
Python
267 lines
9.6 KiB
Python
"""MCP 手机控制 Server（M0：设备列表/截图/点击/滑动）。
|
||
|
||
运行：MCP_ALLOW_WRITE=1 python -m mcp_server.mcp_server
|
||
客户端：Streamable HTTP @ http://<host>:8033/mcp
|
||
"""
|
||
import base64
|
||
import io
|
||
import logging
|
||
|
||
from fastmcp import FastMCP
|
||
from PIL import Image
|
||
|
||
from mcp_server import audit, config
|
||
from mcp_server.platform_client import PlatformClient, PlatformError
|
||
|
||
logging.basicConfig(level=logging.INFO,
|
||
format="%(asctime)s %(levelname)s [%(name)s] %(message)s")
|
||
_log = logging.getLogger("mcp")
|
||
|
||
S = config.Settings()
|
||
|
||
# 坐标空间缓存：serial -> (display_w, display_h, native_w, native_h)
|
||
# de_screenshot 返回的图与 de_tap/de_swipe 的坐标同一空间（display），
|
||
# server 按比例换算为设备原生坐标（模型只感知截图坐标系，不感知原生分辨率）。
|
||
_coord = {}
|
||
audit.init(S.audit_file)
|
||
|
||
mcp = FastMCP("mobile-control")
|
||
_platform = None
|
||
|
||
|
||
def platform():
|
||
"""惰性初始化平台客户端（启动即登录，失败明确报错）。"""
|
||
global _platform
|
||
if _platform is None:
|
||
_platform = PlatformClient(S.platform_url, S.platform_user,
|
||
S.platform_pass, S.platform_timeout)
|
||
return _platform
|
||
|
||
|
||
def _check_serial(serial):
|
||
"""白名单校验：未配置时限制为平台设备池（enabled）设备。"""
|
||
if not serial:
|
||
raise PlatformError("invalid_param", "缺少 serial")
|
||
if S.allowed_serials and serial not in S.allowed_serials:
|
||
raise PlatformError("device_not_allowed", f"设备 {serial} 不在白名单")
|
||
return serial
|
||
|
||
|
||
def _check_write():
|
||
if not S.allow_write:
|
||
raise PlatformError("write_disabled", "写操作未启用（MCP_ALLOW_WRITE=1 开启）")
|
||
|
||
|
||
def _to_native(serial, x, y):
|
||
"""截图坐标 → 设备原生坐标（按最近一次截图的比例换算）。"""
|
||
c = _coord.get(serial)
|
||
if not c:
|
||
raise PlatformError("invalid_param",
|
||
"请先对该设备执行 de_screenshot（需要建立坐标空间）")
|
||
dw, dh, nw, nh = c
|
||
return (round(x * nw / dw), round(y * nh / dh))
|
||
|
||
|
||
def _err(e: PlatformError):
|
||
return {"ok": False, "error": {"code": e.code, "message": e.message}}
|
||
|
||
|
||
def _ok(data):
|
||
return {"ok": True, "data": data}
|
||
|
||
|
||
@mcp.tool()
|
||
def de_list_devices() -> dict:
|
||
"""列出可控制设备：serial/在线状态/型号/任务状态/前台 App。
|
||
|
||
返回 [{serial, model, online, task_job, worker_status, foreground_app}]。
|
||
"""
|
||
try:
|
||
devs = platform().list_devices()
|
||
except PlatformError as e:
|
||
return _err(e)
|
||
out = []
|
||
for d in devs:
|
||
out.append({
|
||
"serial": d.get("serial"),
|
||
"model": d.get("model") or d.get("device_name") or "",
|
||
"online": bool(d.get("present")),
|
||
"task_job": d.get("task_job") or "",
|
||
"worker_status": d.get("worker_status") or "idle",
|
||
"foreground_app": d.get("foreground_app") or "",
|
||
})
|
||
audit.audit("de_list_devices", "", "", f"{len(out)} 台")
|
||
return _ok(out)
|
||
|
||
|
||
@mcp.tool()
|
||
def de_screenshot(serial: str) -> dict:
|
||
"""截取设备屏幕并返回图像（image/jpeg，宽 ≤540px）。
|
||
|
||
同时返回 {width, height, screen_state}。多模态客户端可直接看图。
|
||
"""
|
||
try:
|
||
serial = _check_serial(serial)
|
||
jpeg, screen_state = platform().screenshot(serial)
|
||
img = Image.open(io.BytesIO(jpeg))
|
||
w, h = img.size
|
||
if w > S.screenshot_width:
|
||
ratio = S.screenshot_width / w
|
||
img = img.resize((S.screenshot_width, int(h * ratio)))
|
||
buf = io.BytesIO()
|
||
img.convert("RGB").save(buf, "JPEG", quality=S.jpeg_quality)
|
||
data = base64.b64encode(buf.getvalue()).decode()
|
||
# 记录坐标空间（display=返回图尺寸，native=设备原生），供 tap/swipe 换算
|
||
nw, nh = platform().screen_size(serial)
|
||
_coord[serial] = (img.size[0], img.size[1], nw, nh)
|
||
except PlatformError as e:
|
||
return _err(e)
|
||
except Exception as e:
|
||
return _err(PlatformError("platform_unavailable", f"截图处理失败: {e}"))
|
||
audit.audit("de_screenshot", serial, f"{w}x{h}", "ok")
|
||
dw, dh, nw, nh = _coord[serial]
|
||
return _ok({"image": {"type": "image", "data": data,
|
||
"mimeType": "image/jpeg"},
|
||
"width": dw, "height": dh,
|
||
"native_size": {"width": nw, "height": nh},
|
||
"screen_state": screen_state})
|
||
|
||
|
||
@mcp.tool()
|
||
def de_tap(serial: str, x: int, y: int) -> dict:
|
||
"""点击设备屏幕指定坐标。
|
||
|
||
坐标空间 = de_screenshot 返回的图像坐标（display 空间）——先截图拿到
|
||
native_size 后再点击，server 自动换算为设备原生坐标。
|
||
"""
|
||
try:
|
||
_check_write()
|
||
serial = _check_serial(serial)
|
||
if x < 0 or y < 0:
|
||
raise PlatformError("invalid_param", "坐标不能为负")
|
||
nx, ny = _to_native(serial, x, y)
|
||
platform().tap(serial, nx, ny)
|
||
except PlatformError as e:
|
||
return _err(e)
|
||
audit.audit("de_tap", serial, f"({x},{y})->native({nx},{ny})", "ok")
|
||
return _ok({"action": "tap", "serial": serial, "x": x, "y": y})
|
||
|
||
|
||
@mcp.tool()
|
||
def de_swipe(serial: str, x1: int, y1: int, x2: int, y2: int,
|
||
duration: float = 0.2) -> dict:
|
||
"""在设备屏幕上滑动（坐标空间同 de_tap：截图坐标，server 换算原生）。"""
|
||
try:
|
||
_check_write()
|
||
serial = _check_serial(serial)
|
||
nx1, ny1 = _to_native(serial, x1, y1)
|
||
nx2, ny2 = _to_native(serial, x2, y2)
|
||
platform().swipe(serial, nx1, ny1, nx2, ny2, duration)
|
||
except PlatformError as e:
|
||
return _err(e)
|
||
audit.audit("de_swipe", serial, f"({x1},{y1})->({x2},{y2})", "ok")
|
||
return _ok({"action": "swipe", "serial": serial})
|
||
|
||
|
||
_KEYS = ("back", "home", "recent", "menu", "power", "volume_up",
|
||
"volume_down", "enter", "delete", "search", "camera")
|
||
|
||
|
||
@mcp.tool()
|
||
def de_ui_tree(serial: str) -> dict:
|
||
"""获取当前界面元素树（文本 JSON）：每元素含 text/resource_id/description/class/bounds。
|
||
|
||
优先用它定位元素（元素驱动操作），比纯坐标点击更可靠。
|
||
"""
|
||
try:
|
||
serial = _check_serial(serial)
|
||
els = platform().ui_elements(serial)
|
||
except PlatformError as e:
|
||
return _err(e)
|
||
# 精简输出：去掉 suggested/深度噪音，保留可定位属性
|
||
slim = []
|
||
for e in els:
|
||
slim.append({
|
||
"text": e.get("text", "")[:50],
|
||
"id": e.get("resource_id", "")[:80],
|
||
"desc": e.get("description", "")[:50],
|
||
"class": e.get("class", "").split(".")[-1],
|
||
"bounds": e.get("bounds", ""),
|
||
})
|
||
audit.audit("de_ui_tree", serial, "", f"{len(slim)} 元素")
|
||
return _ok({"count": len(slim), "elements": slim[:300]})
|
||
|
||
|
||
@mcp.tool()
|
||
def de_tap_element(serial: str, by: str, value: str, index: int = 1) -> dict:
|
||
"""按元素点击（不需要坐标）：by=text|id|desc，value 为匹配文本/资源 id/描述。
|
||
|
||
元素驱动操作比坐标可靠（界面变化自适应）；元素不存在时返回错误，
|
||
可改用 de_ui_tree 查元素或 de_tap 坐标兜底。index 用于多命中取第几个（默认 1）。
|
||
"""
|
||
try:
|
||
_check_write()
|
||
serial = _check_serial(serial)
|
||
if by not in ("text", "id", "desc"):
|
||
raise PlatformError("invalid_param", "by 可选 text/id/desc")
|
||
if not value or index < 1:
|
||
raise PlatformError("invalid_param", "value 不能为空且 index>=1")
|
||
import uiautomator2 as u2
|
||
d = u2.connect(serial)
|
||
kw = {"text": value} if by == "text" else (
|
||
{"resourceId": value} if by == "id" else {"description": value})
|
||
if index > 1:
|
||
kw["instance"] = index - 1
|
||
el = d(**kw)
|
||
if not el.exists:
|
||
raise PlatformError("device_offline",
|
||
f"未找到元素（{by}={value}，index={index}）——"
|
||
f"建议 de_ui_tree 查看实际元素或 de_tap 用坐标")
|
||
el.click()
|
||
except PlatformError as e:
|
||
return _err(e)
|
||
except Exception as e:
|
||
return _err(PlatformError("platform_unavailable",
|
||
f"元素点击失败: {type(e).__name__}: {str(e)[:100]}"))
|
||
audit.audit("de_tap_element", serial, f"{by}={value}#{index}", "ok")
|
||
return _ok({"action": "tap_element", "serial": serial,
|
||
"by": by, "value": value, "index": index})
|
||
|
||
|
||
@mcp.tool()
|
||
def de_wake(serial: str) -> dict:
|
||
"""点亮设备屏幕并解锁（熄屏时先调用它再截图）。"""
|
||
try:
|
||
_check_write()
|
||
serial = _check_serial(serial)
|
||
platform().wake(serial)
|
||
except PlatformError as e:
|
||
return _err(e)
|
||
audit.audit("de_wake", serial, "", "ok")
|
||
return _ok({"action": "wake", "serial": serial})
|
||
|
||
|
||
@mcp.tool()
|
||
def de_press_key(serial: str, key: str) -> dict:
|
||
"""按设备按键：back/home/recent/menu/power/enter/delete 等。"""
|
||
try:
|
||
_check_write()
|
||
serial = _check_serial(serial)
|
||
if key not in _KEYS:
|
||
raise PlatformError("invalid_param", f"不支持的按键: {key}（可选 {_KEYS}）")
|
||
platform().press_key(serial, key)
|
||
except PlatformError as e:
|
||
return _err(e)
|
||
audit.audit("de_press_key", serial, key, "ok")
|
||
return _ok({"action": "press_key", "serial": serial, "key": key})
|
||
|
||
|
||
def main():
|
||
_log.info("MCP mobile-control 启动: allow_write=%s port=%s",
|
||
S.allow_write, S.http_port)
|
||
mcp.run(transport="http", host=S.http_host, port=S.http_port)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|