fix: AI 点击精度优化——①坐标吸附:/api/screen/tap 加 snap=1,MCP de_tap 固定开启(dump UI 树找包含点击点的最小可点击元素点中心,模型坐标偏 20-50px 也点得准,大屏触控不带 snap 行为不变);②新 de_tap_text 语义点击:按屏幕可见文字一次完成找+点(UI 树 textContains/descriptionContains → OCR 中心兜底,WebView/图片文字也能点);③de_tap_element 支持 text_contains/desc_contains 模糊匹配;④de_ui_tree 可点击元素优先 + limit 参数防 token 膨胀;⑤agent 提示词重写:文字语义点击优先、坐标仅纯图形兜底且自动吸附、点击后无变化禁止重复同坐标
This commit is contained in:
@@ -93,12 +93,32 @@ class PlatformClient:
|
||||
raise PlatformError("device_offline", str(j.get("error", "取分辨率失败"))[:120])
|
||||
return int(j["width"]), int(j["height"])
|
||||
|
||||
def tap(self, serial, x, y):
|
||||
"""点击(POST /api/screen/tap)。"""
|
||||
def tap(self, serial, x, y, snap=False):
|
||||
"""点击(POST /api/screen/tap)。
|
||||
|
||||
snap=True:点落在可点击元素内则吸附到元素中心(AI 粗略坐标也能点准)。
|
||||
返回平台 JSON(含 snapped/x/y/label)。
|
||||
"""
|
||||
r = self._post("/api/screen/tap", json={"serial": serial,
|
||||
"x": int(x), "y": int(y)})
|
||||
"x": int(x), "y": int(y),
|
||||
"snap": 1 if snap else 0})
|
||||
return self._check_op(r, "tap")
|
||||
|
||||
def tap_text(self, serial, text):
|
||||
"""按屏幕文字点击(平台解析:UI 树子串匹配 → OCR 兜底)。
|
||||
|
||||
返回 {ok, found, method, matched, x, y}——found=false 是业务结果
|
||||
(屏幕无该文字),非设备错误;设备离线/不可达仍抛 PlatformError。
|
||||
"""
|
||||
r = self._post("/api/screen/tap_text",
|
||||
json={"serial": serial, "text": str(text)})
|
||||
if r.status_code == 503:
|
||||
raise PlatformError("device_offline", r.text[:120])
|
||||
if r.status_code != 200:
|
||||
raise PlatformError("platform_unavailable",
|
||||
f"tap_text HTTP {r.status_code}: {r.text[:120]}")
|
||||
return r.json() or {}
|
||||
|
||||
def swipe(self, serial, x1, y1, x2, y2, duration=0.2):
|
||||
"""滑动(POST /api/screen/swipe)。"""
|
||||
r = self._post("/api/screen/swipe", json={
|
||||
@@ -147,27 +167,6 @@ class PlatformClient:
|
||||
"schedule": (t.get("schedule") or {}).get("mode", "")})
|
||||
return tasks
|
||||
|
||||
def sleep(self, serial):
|
||||
"""熄屏(POST /api/device/screen_all mode=off)。"""
|
||||
r = self._post("/api/device/screen_all",
|
||||
json={"mode": "off", "serials": [serial]})
|
||||
return self._check_op(r, "sleep")
|
||||
|
||||
def list_tasks(self):
|
||||
"""任务计划列表(GET /api/jobs)。"""
|
||||
r = self._get("/api/jobs")
|
||||
if r.status_code != 200:
|
||||
raise PlatformError("platform_unavailable",
|
||||
f"/api/jobs HTTP {r.status_code}")
|
||||
j = r.json() or {}
|
||||
tasks = []
|
||||
for t in j.get("jobs") or []:
|
||||
tasks.append({"id": t.get("id"), "name": t.get("name"),
|
||||
"task_type": t.get("task_type"),
|
||||
"enabled": t.get("enabled"),
|
||||
"schedule": (t.get("schedule") or {}).get("mode", "")})
|
||||
return tasks
|
||||
|
||||
def press_key(self, serial, key):
|
||||
"""按键(POST /api/screen/key)。"""
|
||||
r = self._post("/api/screen/key",
|
||||
|
||||
Reference in New Issue
Block a user