diff --git a/core/uiauto_helper.py b/core/uiauto_helper.py index a0d5140..76c58e6 100644 --- a/core/uiauto_helper.py +++ b/core/uiauto_helper.py @@ -12,6 +12,7 @@ API 参考(uiautodev 0.14): - GET /api/android/{serial}/dump_hierarchy — 元素树 JSON """ import requests +from collections import Counter from core.logger import get_logger @@ -78,12 +79,23 @@ def get_elements(serial): """获取指定设备的 UI 元素树。 返回 (ok, data_or_error): - ok=True — data 是元素列表 [{name, attrs:{...}, children:[...]}, ...] + ok=True — data 是元素列表 [{name, attrs..., suggested:{type,value,indexed?,occ?,total?,broad?}}, ...] ok=False — data 是错误消息字符串 - 元素树由 uiautodev 的 U2AndroidDriver.dump_hierarchy 返回(JSON)。 - 我们递归提取每个节点的关键属性(resource-id/text/description/class/...), - 供前端列表展示和选择。 + 元素树由 uiautodev 的 dump_hierarchy 返回(JSON)。我们递归提取每个节点的 + 关键属性(resource-id/text/description/class/bounds...),供前端列表展示和选择。 + + 选择器精度策略(精确到具体按钮的关键): + 1. 有 resource-id/text/content-desc 的元素: + 先预统计该属性在整棵树中的出现次数—— + - 唯一出现:直接用属性选择器 //*[@resource-id="x"] + - 重复出现(如抖音信息流点赞按钮同 id 几十个):附加 [k] 位置谓词 + 精确到具体实例 //*[@resource-id="x"][k]。uiautomator2 的 d.xpath() + 底层是 lxml 标准 XPath,[k] 与抓取时同一语义,不会误中屏幕外第一个。 + 2. 无任何属性的元素: + 用最近一个有属性祖先的选择器限定范围 + 同 class 兄弟序号定位 + (如 //*[@resource-id="x"]/FrameLayout/ImageView[2]); + 整棵树都没有属性时退化为从根开始的结构路径(标记 broad,前端提示脆弱)。 """ if not serial: return False, "缺少 serial" @@ -99,7 +111,7 @@ def get_elements(serial): # uiautodev hierarchy 返回 Node 树:{key, name, bounds, rect, properties, children} # 提取成扁平的可选列表(保留层级缩进信息) elements = [] - _flatten(data, elements, depth=0) + _extract(data, elements) if not elements: return False, "当前界面未抓取到元素" return True, elements @@ -112,78 +124,117 @@ def get_elements(serial): return False, f"抓取失败: {e}" -def _flatten(node, out, depth=0, path="", ctx=None, tag_index=0): - """递归扁平化元素树,提取关键属性供前端选择。 +def _xpath_q(v): + """XPath 字符串字面量:优先双引号,值含双引号时改用单引号包裹(XPath 1.0 无转义)。""" + if '"' in v: + return "'" + v + "'" + return '"' + v + '"' - uiautodev dump_hierarchy 返回 Node 格式: - {key, name, bounds(归一化浮点), rect({x,y,width,height}), properties({所有属性}), children} - 其中 properties 包含原始 XML 属性(resource-id/text/content-desc/class/bounds 字符串等)。 - ctx: 最近一个有唯一属性(resource-id/text/content-desc)祖先的 xpath。 - 无唯一属性的元素用它限定范围生成精确选择器,避免裸 class(如 //ImageView)过宽误点。 - tag_index: 本节点在父节点同 class 兄弟中的序号(1-based),用于消歧。 - """ - if not isinstance(node, dict): - return - props = node.get("properties") or {} - name = node.get("name") or props.get("class") or "" - # bounds:优先取 properties 中的原始字符串 "[x1,y1][x2,y2]"(前端渲染 overlay 用) - bounds_str = props.get("bounds", "") - if not bounds_str and node.get("rect"): - r = node["rect"] - bounds_str = f"[{r['x']},{r['y']}][{r['x']+r['width']},{r['y']+r['height']}]" +def _extract(root, out): + """把 uiautodev 元素树扁平化为可选列表,并为每个节点生成精确选择器建议。""" + # 全树属性出现次数(预统计,供重复元素加 [k] 序号消歧) + id_cnt, text_cnt, desc_cnt = Counter(), Counter(), Counter() + # 文档顺序已出现次数(决定当前元素是第几个) + seen_id, seen_text, seen_desc = Counter(), Counter(), Counter() - rid = props.get("resource-id", "") - text = props.get("text", "") - desc = props.get("content-desc", "") - cls = props.get("class", "") - tag = cls.split(".")[-1] if cls else "*" + def count_attrs(node): + if not isinstance(node, dict): + return + props = node.get("properties") or {} + rid, text, desc = (props.get(k, "") for k in ("resource-id", "text", "content-desc")) + if rid: + id_cnt[rid] += 1 + if text: + text_cnt[text] += 1 + if desc: + desc_cnt[desc] += 1 + for c in node.get("children") or []: + count_attrs(c) - # 推荐选择器:唯一属性优先;否则用最近锚点祖先限定范围(带同class序号消歧) - if rid: - suggested = {"type": "xpath", "value": f'//*[@resource-id="{rid}"]'} - child_ctx = suggested["value"] - elif text: - suggested = {"type": "xpath", "value": f'//*[@text="{text}"]'} - child_ctx = suggested["value"] - elif desc: - suggested = {"type": "xpath", "value": f'//*[@content-desc="{desc}"]'} - child_ctx = suggested["value"] - else: - if ctx: - seg = f"/{tag}" + (f"[{tag_index}]" if tag_index > 1 else "") - suggested = {"type": "xpath", "value": f"{ctx}{seg}"} - child_ctx = f"{ctx}{seg}" + def attr_selector(attr, value, cnt, seen): + """属性选择器:唯一直接出,重复加 [k] 位置谓词。返回 (选择器, suggested)。""" + seen[value] += 1 + val = f'//*[@{attr}={_xpath_q(value)}]' + if cnt[value] > 1: + val += f"[{seen[value]}]" + return val, {"type": "xpath", "value": val, + "indexed": True, "occ": seen[value], "total": cnt[value]} + return val, {"type": "xpath", "value": val} + + def flatten(node, depth=0, path="", ctx=None, tag_path="", tag_index=1): + """递归扁平化。 + + ctx: 最近一个有属性祖先的选择器(含序号),无属性元素用它限定范围。 + tag_path: 从根到本节点的完整结构路径(class + 同class兄弟序号),兜底用。 + tag_index: 本节点在父下同 class 兄弟中的序号(1-based)。 + """ + if not isinstance(node, dict): + return + props = node.get("properties") or {} + name = node.get("name") or props.get("class") or "" + # bounds:优先取 properties 中的原始字符串 "[x1,y1][x2,y2]"(前端渲染 overlay 用) + bounds_str = props.get("bounds", "") + if not bounds_str and node.get("rect"): + # rect 兜底:只有整数像素坐标才用;uiautodev 归一化浮点坐标无法换算像素 + r = node["rect"] + try: + vals = [r["x"], r["y"], r["x"] + r["width"], r["y"] + r["height"]] + if all(isinstance(v, (int, float)) and float(v).is_integer() for v in vals): + bounds_str = f"[{int(vals[0])},{int(vals[1])}][{int(vals[2])},{int(vals[3])}]" + except (TypeError, KeyError): + pass + + rid = props.get("resource-id", "") + text = props.get("text", "") + desc = props.get("content-desc", "") + cls = props.get("class", "") + tag = cls.split(".")[-1] if cls else "*" + # 始终带同 class 兄弟序号(含 [1]),结构路径才精确无歧义 + seg = f"{tag}[{tag_index}]" + own_tag_path = f"{tag_path}/{seg}" if tag_path else seg + + # ---- 推荐选择器:唯一属性 > 锚点祖先限定 > 全结构路径 ---- + if rid: + child_ctx, suggested = attr_selector("resource-id", rid, id_cnt, seen_id) + elif text: + child_ctx, suggested = attr_selector("text", text, text_cnt, seen_text) + elif desc: + child_ctx, suggested = attr_selector("content-desc", desc, desc_cnt, seen_desc) + elif ctx: + val = f"{ctx}/{seg}" + suggested = {"type": "xpath", "value": val} + child_ctx = val else: - # 无唯一属性且无锚点祖先:裸 class 过宽,标记 broad 让前端提示 - suggested = {"type": "xpath", "value": f"//{tag}", "broad": True} + # 无唯一属性且无锚点祖先:用从根开始的结构路径(脆弱,标记 broad 让前端提示) + suggested = {"type": "xpath", "value": f"//{own_tag_path}", "broad": True} child_ctx = None - item = { - "depth": depth, - "path": path, - "name": name, - "resource_id": rid, - "text": text, - "description": desc, - "class": cls, - "package": props.get("package", ""), - "clickable": props.get("clickable", ""), - "bounds": bounds_str, - # 推荐选择器(resource-id > text > description > 锚点祖先限定class) - "suggested": suggested, - } - out.append(item) - # 递归子节点,同时计算每个子节点在父下同 class 兄弟中的序号 - children = node.get("children") or [] - for i, child in enumerate(children): - cprops = child.get("properties") or {} - ccls = cprops.get("class", "") - ctag = ccls.split(".")[-1] if ccls else "*" - same = 1 - for prev in children[:i]: - pcls = (prev.get("properties") or {}).get("class", "") - ptag = pcls.split(".")[-1] if pcls else "*" - if ptag == ctag: - same += 1 - _flatten(child, out, depth + 1, f"{path}/{i}", child_ctx, same) + out.append({ + "depth": depth, + "path": path, + "name": name, + "resource_id": rid, + "text": text, + "description": desc, + "class": cls, + "package": props.get("package", ""), + "clickable": props.get("clickable", ""), + "bounds": bounds_str, + "suggested": suggested, + }) + + # 递归子节点,同时计算每个子节点在父下同 class 兄弟中的序号 + children = node.get("children") or [] + for i, child in enumerate(children): + cprops = child.get("properties") or {} + ccls = cprops.get("class", "") + ctag = ccls.split(".")[-1] if ccls else "*" + same = 1 + sum( + 1 for prev in children[:i] + if ((prev.get("properties") or {}).get("class", "")).split(".")[-1] == ctag + ) + flatten(child, depth + 1, f"{path}/{i}", child_ctx, own_tag_path, same) + + count_attrs(root) + flatten(root) diff --git a/static/admin/monitor.js b/static/admin/monitor.js index 1fec537..7f8a4cd 100644 --- a/static/admin/monitor.js +++ b/static/admin/monitor.js @@ -1397,7 +1397,7 @@ var _stepEditor={ '
'+ '
'+ '
'+ - ''+ + ''+ '加载中...'+ '
'+ '
'+ @@ -1412,21 +1412,35 @@ var _stepEditor={ document.getElementById('el-picker-footer').innerHTML= ''+ ''; - // 加载截图和元素树 + // 加载截图和元素树(_refreshShot 内部同时刷新两者) self._refreshShot(); - fetch('/api/uiauto/elements?serial='+encodeURIComponent(serial)) + }, + // 拉取元素树(进入页面和刷新时共用;带时间戳防缓存) + _loadElData:function(){ + var serial=this._elSerial; + if(!serial)return; + var listEl=document.getElementById('el-list'); + if(listEl)listEl.innerHTML='
加载元素树...
'; + var self=this; + fetch('/api/uiauto/elements?serial='+encodeURIComponent(serial)+'&_='+Date.now()) .then(function(r){return r.json();}) .then(function(data){ + var le=document.getElementById('el-list'); + if(!le)return; if(!data.ok){ - document.getElementById('el-list').innerHTML='
'+esc(data.error||'抓取失败')+'
'; + le.innerHTML='
'+esc(data.error||'抓取失败')+'
'; return; } self._elData=data.elements||[]; self._renderEl(self._elData); + // 刷新后保持搜索过滤状态 + var search=document.getElementById('el-search'); + if(search&&search.value)self._filterEl(search.value); self._renderOverlays(); }) .catch(function(e){ - document.getElementById('el-list').innerHTML='
请求失败: '+esc(e)+'
'; + var le=document.getElementById('el-list'); + if(le)le.innerHTML='
请求失败: '+esc(e)+'
'; }); }, // 刷新截图 @@ -1454,6 +1468,8 @@ var _stepEditor={ if(infoEl)infoEl.textContent='失败'; }; img.src='/api/uiauto/screenshot?serial='+encodeURIComponent(serial)+'&_='+Date.now(); + // 同时重新拉取元素树——只刷截图会导致"截图是新的、元素框是旧的"错位 + this._loadElData(); }, // 在截图上渲染元素边界框(可点击) _renderOverlays:function(){ @@ -1531,6 +1547,12 @@ var _stepEditor={ item.classList.add('active'); item.scrollIntoView({block:'nearest',behavior:'smooth'}); } + // 信息栏显示将填入的选择器,确认是否精确命中目标 + var infoEl=document.getElementById('el-shot-info'); + if(infoEl&&this._elData&&this._elData[i]){ + var sg=this._elData[i].suggested||{}; + infoEl.textContent='已选中: '+(sg.type||'')+' '+(sg.value||''); + } }, _renderEl:function(list){ var self=this; @@ -1541,12 +1563,14 @@ var _stepEditor={ if(el.text)attrs.push('text:'+esc(el.text)+''); if(el.description)attrs.push('desc:'+esc(el.description)+''); var indent=' '.repeat(Math.min(el.depth||0,6)); - return '
'+ + var sg=el.suggested||{}; + return '
'+ ''+(el.depth||0)+''+ '
'+indent+esc(el.name||el.class||'')+'
'+ '
'+attrs.join('')+'
'+ - ''+esc(el.suggested.type)+''+ - (el.suggested&&el.suggested.broad?' ⚠宽泛':'')+ + ''+esc(sg.type)+''+ + (sg.indexed?' #'+sg.occ+'/'+sg.total+'':'')+ + (sg.broad?' ⚠宽泛':'')+ '
'; }).join(''); document.getElementById('el-list').innerHTML=html||'
无元素
'; @@ -1834,17 +1858,20 @@ async function loadUsers(){ filter:(u,kw)=>(u.username||'').toLowerCase().includes(kw), render:renderUserRow, tbody:'tb-users',pager:'pager-users',pageSize:10, - empty:'暂无用户' + empty:'暂无用户' }); applyListPager('users'); initListSearch('users','search-users'); } function renderUserRow(u){ const admin=u.is_admin?'管理员':'普通'; + const perms=(u.perms||[]).map(p=>''+(PERM_LABELS[p]||p)+'').join(' ') + ||'-'; return ''+ ''+u.id+''+ ''+esc(u.username)+''+ ''+admin+''+ + ''+perms+''+ ''+ ''+ (u.username==='admin'?'':'')+ diff --git a/web_server.py b/web_server.py index 93ad230..34d7faf 100644 --- a/web_server.py +++ b/web_server.py @@ -596,18 +596,73 @@ def api_uiauto_status(): return jsonify({"ok": True, "running": uiauto_helper.is_running()}) +@app.route("/api/steps/test", methods=["POST"]) +@perm_required(PERM_DEVICES) +def api_steps_test(): + """测试单个步骤:在指定设备上试执行,验证选择器是否命中(步骤编辑器"测试此步骤")。 + + 请求: {"serial": "100.100.10.11:5555", "step": {"type": "click", "params": {...}}} + 只读连接(adb connect + u2),不占用/释放 STF,与运行中任务互不干扰。 + 返回 result: "命中" / "未找到" / "已执行"。 + """ + data = request.json or {} + serial = (data.get("serial") or "").strip() + step = data.get("step") or {} + if not serial: + return jsonify({"ok": False, "error": "缺少 serial"}), 400 + if not step.get("type"): + return jsonify({"ok": False, "error": "缺少步骤类型"}), 400 + from tasks.generic.task import test_step + ok, msg, result = test_step(serial, step) + res_txt = {True: "命中", False: "未找到"}.get(result, "已执行") + _log.info(f"测试步骤 {step.get('type')} @ {serial}: {res_txt} ({msg})") + return jsonify({"ok": ok, "msg": msg, "result": res_txt}) + + @app.route("/api/uiauto/devices") -@login_required +@perm_required(PERM_DEVICES) def api_uiauto_devices(): - """获取 uiauto2 已连接的设备列表(供抓取元素时选择设备)。""" + """获取可选设备列表(抓取元素用)。 + + uiautodev 只认识本地 adb 已连接的设备;平台设备池(STF)里在线但未 + 本地连接的设备(如 100.100.10.10)在这里补全,并先做一次轻量 adb + connect(单次尝试,绝不 kill-server),让它们可被 uiautodev 抓取。 + """ ok, data = uiauto_helper.list_devices() - if ok: - return jsonify({"ok": True, "devices": data}) - return jsonify({"ok": False, "error": data}), 503 + if not ok: + return jsonify({"ok": False, "error": data}), 503 + seen = {d.get("serial") for d in data} + try: + pool = stf.list_all_devices() + except Exception: + pool = [] + added = 0 + for d in pool: + serial = d.get("serial", "") + if not serial or serial in seen: + continue + if not (d.get("present") and d.get("ready")): + continue + # 轻量连接(单次尝试,不重试不 kill-server);连不上也照样列出, + # 前端抓取时会报明确错误,不影响其他设备的连接 + adb_connect_light(serial) + data.append({ + "serial": serial, + "model": d.get("model") or d.get("product") or "", + "product": d.get("product") or "", + "name": serial, + "status": "device", + "enabled": True, + }) + seen.add(serial) + added += 1 + if added: + _log.info(f"抓取元素设备列表补全 {added} 台 STF 在线设备: {list(seen)}") + return jsonify({"ok": True, "devices": data}) @app.route("/api/uiauto/screenshot") -@login_required +@perm_required(PERM_DEVICES) def api_uiauto_screenshot(): """通过 uiauto2 获取设备截图(供抓取元素时显示设备画面)。