feat: 抓取元素选择器精度优化 + 设备列表补全 + 刷新同步
This commit is contained in:
+124
-73
@@ -12,6 +12,7 @@ API 参考（uiautodev 0.14）：
|
||||
- GET /api/android/{serial}/dump_hierarchy — 元素树 JSON
|
||||
"""
|
||||
import requests
|
||||
from collections import Counter
|
||||
|
||||
from core.logger import get_logger
|
||||
|
||||
@@ -78,12 +79,23 @@ def get_elements(serial):
|
||||
"""获取指定设备的 UI 元素树。
|
||||
|
||||
返回 (ok, data_or_error)：
|
||||
ok=True — data 是元素列表 [{name, attrs:{...}, children:[...]}, ...]
|
||||
ok=True — data 是元素列表 [{name, attrs..., suggested:{type,value,indexed?,occ?,total?,broad?}}, ...]
|
||||
ok=False — data 是错误消息字符串
|
||||
|
||||
元素树由 uiautodev 的 U2AndroidDriver.dump_hierarchy 返回（JSON）。
|
||||
我们递归提取每个节点的关键属性（resource-id/text/description/class/...），
|
||||
供前端列表展示和选择。
|
||||
元素树由 uiautodev 的 dump_hierarchy 返回（JSON）。我们递归提取每个节点的
|
||||
关键属性（resource-id/text/description/class/bounds...），供前端列表展示和选择。
|
||||
|
||||
选择器精度策略（精确到具体按钮的关键）：
|
||||
1. 有 resource-id/text/content-desc 的元素：
|
||||
先预统计该属性在整棵树中的出现次数——
|
||||
- 唯一出现：直接用属性选择器 //*[@resource-id="x"]
|
||||
- 重复出现（如抖音信息流点赞按钮同 id 几十个）：附加 [k] 位置谓词
|
||||
精确到具体实例 //*[@resource-id="x"][k]。uiautomator2 的 d.xpath()
|
||||
底层是 lxml 标准 XPath，[k] 与抓取时同一语义，不会误中屏幕外第一个。
|
||||
2. 无任何属性的元素：
|
||||
用最近一个有属性祖先的选择器限定范围 + 同 class 兄弟序号定位
|
||||
（如 //*[@resource-id="x"]/FrameLayout/ImageView[2]）；
|
||||
整棵树都没有属性时退化为从根开始的结构路径（标记 broad，前端提示脆弱）。
|
||||
"""
|
||||
if not serial:
|
||||
return False, "缺少 serial"
|
||||
@@ -99,7 +111,7 @@ def get_elements(serial):
|
||||
# uiautodev hierarchy 返回 Node 树：{key, name, bounds, rect, properties, children}
|
||||
# 提取成扁平的可选列表（保留层级缩进信息）
|
||||
elements = []
|
||||
_flatten(data, elements, depth=0)
|
||||
_extract(data, elements)
|
||||
if not elements:
|
||||
return False, "当前界面未抓取到元素"
|
||||
return True, elements
|
||||
@@ -112,78 +124,117 @@ def get_elements(serial):
|
||||
return False, f"抓取失败: {e}"
|
||||
|
||||
|
||||
def _flatten(node, out, depth=0, path="", ctx=None, tag_index=0):
|
||||
"""递归扁平化元素树，提取关键属性供前端选择。
|
||||
def _xpath_q(v):
|
||||
"""XPath 字符串字面量：优先双引号，值含双引号时改用单引号包裹（XPath 1.0 无转义）。"""
|
||||
if '"' in v:
|
||||
return "'" + v + "'"
|
||||
return '"' + v + '"'
|
||||
|
||||
uiautodev dump_hierarchy 返回 Node 格式：
|
||||
{key, name, bounds(归一化浮点), rect({x,y,width,height}), properties({所有属性}), children}
|
||||
其中 properties 包含原始 XML 属性（resource-id/text/content-desc/class/bounds 字符串等）。
|
||||
|
||||
ctx: 最近一个有唯一属性（resource-id/text/content-desc）祖先的 xpath。
|
||||
无唯一属性的元素用它限定范围生成精确选择器，避免裸 class（如 //ImageView）过宽误点。
|
||||
tag_index: 本节点在父节点同 class 兄弟中的序号（1-based），用于消歧。
|
||||
"""
|
||||
if not isinstance(node, dict):
|
||||
return
|
||||
props = node.get("properties") or {}
|
||||
name = node.get("name") or props.get("class") or ""
|
||||
# bounds：优先取 properties 中的原始字符串 "[x1,y1][x2,y2]"（前端渲染 overlay 用）
|
||||
bounds_str = props.get("bounds", "")
|
||||
if not bounds_str and node.get("rect"):
|
||||
r = node["rect"]
|
||||
bounds_str = f"[{r['x']},{r['y']}][{r['x']+r['width']},{r['y']+r['height']}]"
|
||||
def _extract(root, out):
|
||||
"""把 uiautodev 元素树扁平化为可选列表，并为每个节点生成精确选择器建议。"""
|
||||
# 全树属性出现次数（预统计，供重复元素加 [k] 序号消歧）
|
||||
id_cnt, text_cnt, desc_cnt = Counter(), Counter(), Counter()
|
||||
# 文档顺序已出现次数（决定当前元素是第几个）
|
||||
seen_id, seen_text, seen_desc = Counter(), Counter(), Counter()
|
||||
|
||||
rid = props.get("resource-id", "")
|
||||
text = props.get("text", "")
|
||||
desc = props.get("content-desc", "")
|
||||
cls = props.get("class", "")
|
||||
tag = cls.split(".")[-1] if cls else "*"
|
||||
def count_attrs(node):
|
||||
if not isinstance(node, dict):
|
||||
return
|
||||
props = node.get("properties") or {}
|
||||
rid, text, desc = (props.get(k, "") for k in ("resource-id", "text", "content-desc"))
|
||||
if rid:
|
||||
id_cnt[rid] += 1
|
||||
if text:
|
||||
text_cnt[text] += 1
|
||||
if desc:
|
||||
desc_cnt[desc] += 1
|
||||
for c in node.get("children") or []:
|
||||
count_attrs(c)
|
||||
|
||||
# 推荐选择器：唯一属性优先；否则用最近锚点祖先限定范围（带同class序号消歧）
|
||||
if rid:
|
||||
suggested = {"type": "xpath", "value": f'//*[@resource-id="{rid}"]'}
|
||||
child_ctx = suggested["value"]
|
||||
elif text:
|
||||
suggested = {"type": "xpath", "value": f'//*[@text="{text}"]'}
|
||||
child_ctx = suggested["value"]
|
||||
elif desc:
|
||||
suggested = {"type": "xpath", "value": f'//*[@content-desc="{desc}"]'}
|
||||
child_ctx = suggested["value"]
|
||||
else:
|
||||
if ctx:
|
||||
seg = f"/{tag}" + (f"[{tag_index}]" if tag_index > 1 else "")
|
||||
suggested = {"type": "xpath", "value": f"{ctx}{seg}"}
|
||||
child_ctx = f"{ctx}{seg}"
|
||||
def attr_selector(attr, value, cnt, seen):
|
||||
"""属性选择器：唯一直接出，重复加 [k] 位置谓词。返回 (选择器, suggested)。"""
|
||||
seen[value] += 1
|
||||
val = f'//*[@{attr}={_xpath_q(value)}]'
|
||||
if cnt[value] > 1:
|
||||
val += f"[{seen[value]}]"
|
||||
return val, {"type": "xpath", "value": val,
|
||||
"indexed": True, "occ": seen[value], "total": cnt[value]}
|
||||
return val, {"type": "xpath", "value": val}
|
||||
|
||||
def flatten(node, depth=0, path="", ctx=None, tag_path="", tag_index=1):
|
||||
"""递归扁平化。
|
||||
|
||||
ctx: 最近一个有属性祖先的选择器（含序号），无属性元素用它限定范围。
|
||||
tag_path: 从根到本节点的完整结构路径（class + 同class兄弟序号），兜底用。
|
||||
tag_index: 本节点在父下同 class 兄弟中的序号（1-based）。
|
||||
"""
|
||||
if not isinstance(node, dict):
|
||||
return
|
||||
props = node.get("properties") or {}
|
||||
name = node.get("name") or props.get("class") or ""
|
||||
# bounds：优先取 properties 中的原始字符串 "[x1,y1][x2,y2]"（前端渲染 overlay 用）
|
||||
bounds_str = props.get("bounds", "")
|
||||
if not bounds_str and node.get("rect"):
|
||||
# rect 兜底：只有整数像素坐标才用；uiautodev 归一化浮点坐标无法换算像素
|
||||
r = node["rect"]
|
||||
try:
|
||||
vals = [r["x"], r["y"], r["x"] + r["width"], r["y"] + r["height"]]
|
||||
if all(isinstance(v, (int, float)) and float(v).is_integer() for v in vals):
|
||||
bounds_str = f"[{int(vals[0])},{int(vals[1])}][{int(vals[2])},{int(vals[3])}]"
|
||||
except (TypeError, KeyError):
|
||||
pass
|
||||
|
||||
rid = props.get("resource-id", "")
|
||||
text = props.get("text", "")
|
||||
desc = props.get("content-desc", "")
|
||||
cls = props.get("class", "")
|
||||
tag = cls.split(".")[-1] if cls else "*"
|
||||
# 始终带同 class 兄弟序号（含 [1]），结构路径才精确无歧义
|
||||
seg = f"{tag}[{tag_index}]"
|
||||
own_tag_path = f"{tag_path}/{seg}" if tag_path else seg
|
||||
|
||||
# ---- 推荐选择器：唯一属性 > 锚点祖先限定 > 全结构路径 ----
|
||||
if rid:
|
||||
child_ctx, suggested = attr_selector("resource-id", rid, id_cnt, seen_id)
|
||||
elif text:
|
||||
child_ctx, suggested = attr_selector("text", text, text_cnt, seen_text)
|
||||
elif desc:
|
||||
child_ctx, suggested = attr_selector("content-desc", desc, desc_cnt, seen_desc)
|
||||
elif ctx:
|
||||
val = f"{ctx}/{seg}"
|
||||
suggested = {"type": "xpath", "value": val}
|
||||
child_ctx = val
|
||||
else:
|
||||
# 无唯一属性且无锚点祖先：裸 class 过宽，标记 broad 让前端提示
|
||||
suggested = {"type": "xpath", "value": f"//{tag}", "broad": True}
|
||||
# 无唯一属性且无锚点祖先：用从根开始的结构路径（脆弱，标记 broad 让前端提示）
|
||||
suggested = {"type": "xpath", "value": f"//{own_tag_path}", "broad": True}
|
||||
child_ctx = None
|
||||
|
||||
item = {
|
||||
"depth": depth,
|
||||
"path": path,
|
||||
"name": name,
|
||||
"resource_id": rid,
|
||||
"text": text,
|
||||
"description": desc,
|
||||
"class": cls,
|
||||
"package": props.get("package", ""),
|
||||
"clickable": props.get("clickable", ""),
|
||||
"bounds": bounds_str,
|
||||
# 推荐选择器（resource-id > text > description > 锚点祖先限定class）
|
||||
"suggested": suggested,
|
||||
}
|
||||
out.append(item)
|
||||
# 递归子节点，同时计算每个子节点在父下同 class 兄弟中的序号
|
||||
children = node.get("children") or []
|
||||
for i, child in enumerate(children):
|
||||
cprops = child.get("properties") or {}
|
||||
ccls = cprops.get("class", "")
|
||||
ctag = ccls.split(".")[-1] if ccls else "*"
|
||||
same = 1
|
||||
for prev in children[:i]:
|
||||
pcls = (prev.get("properties") or {}).get("class", "")
|
||||
ptag = pcls.split(".")[-1] if pcls else "*"
|
||||
if ptag == ctag:
|
||||
same += 1
|
||||
_flatten(child, out, depth + 1, f"{path}/{i}", child_ctx, same)
|
||||
out.append({
|
||||
"depth": depth,
|
||||
"path": path,
|
||||
"name": name,
|
||||
"resource_id": rid,
|
||||
"text": text,
|
||||
"description": desc,
|
||||
"class": cls,
|
||||
"package": props.get("package", ""),
|
||||
"clickable": props.get("clickable", ""),
|
||||
"bounds": bounds_str,
|
||||
"suggested": suggested,
|
||||
})
|
||||
|
||||
# 递归子节点，同时计算每个子节点在父下同 class 兄弟中的序号
|
||||
children = node.get("children") or []
|
||||
for i, child in enumerate(children):
|
||||
cprops = child.get("properties") or {}
|
||||
ccls = cprops.get("class", "")
|
||||
ctag = ccls.split(".")[-1] if ccls else "*"
|
||||
same = 1 + sum(
|
||||
1 for prev in children[:i]
|
||||
if ((prev.get("properties") or {}).get("class", "")).split(".")[-1] == ctag
|
||||
)
|
||||
flatten(child, depth + 1, f"{path}/{i}", child_ctx, own_tag_path, same)
|
||||
|
||||
count_attrs(root)
|
||||
flatten(root)
|
||||
|
||||
+36
-9
@@ -1397,7 +1397,7 @@ var _stepEditor={
|
||||
'<div class="el-picker-main">'+
|
||||
'<div class="el-shot-wrap">'+
|
||||
'<div class="el-shot-toolbar">'+
|
||||
'<button class="btn" onclick="_stepEditor._refreshShot()">刷新截图</button>'+
|
||||
'<button class="btn" onclick="_stepEditor._refreshShot()" title="重新拉取截图和元素树（界面变化后刷新用）">刷新截图+元素</button>'+
|
||||
'<span class="el-shot-info" id="el-shot-info">加载中...</span>'+
|
||||
'</div>'+
|
||||
'<div class="el-shot-container" id="el-shot-container">'+
|
||||
@@ -1412,21 +1412,35 @@ var _stepEditor={
|
||||
document.getElementById('el-picker-footer').innerHTML=
|
||||
'<button class="btn" onclick="_stepEditor._showDevicePicker()">← 返回选设备</button>'+
|
||||
'<button class="btn" onclick="closeElPicker()">取消</button>';
|
||||
// 加载截图和元素树
|
||||
// 加载截图和元素树（_refreshShot 内部同时刷新两者）
|
||||
self._refreshShot();
|
||||
fetch('/api/uiauto/elements?serial='+encodeURIComponent(serial))
|
||||
},
|
||||
// 拉取元素树（进入页面和刷新时共用；带时间戳防缓存）
|
||||
_loadElData:function(){
|
||||
var serial=this._elSerial;
|
||||
if(!serial)return;
|
||||
var listEl=document.getElementById('el-list');
|
||||
if(listEl)listEl.innerHTML='<div class="se-empty">加载元素树...</div>';
|
||||
var self=this;
|
||||
fetch('/api/uiauto/elements?serial='+encodeURIComponent(serial)+'&_='+Date.now())
|
||||
.then(function(r){return r.json();})
|
||||
.then(function(data){
|
||||
var le=document.getElementById('el-list');
|
||||
if(!le)return;
|
||||
if(!data.ok){
|
||||
document.getElementById('el-list').innerHTML='<div class="se-empty">'+esc(data.error||'抓取失败')+'</div>';
|
||||
le.innerHTML='<div class="se-empty">'+esc(data.error||'抓取失败')+'</div>';
|
||||
return;
|
||||
}
|
||||
self._elData=data.elements||[];
|
||||
self._renderEl(self._elData);
|
||||
// 刷新后保持搜索过滤状态
|
||||
var search=document.getElementById('el-search');
|
||||
if(search&&search.value)self._filterEl(search.value);
|
||||
self._renderOverlays();
|
||||
})
|
||||
.catch(function(e){
|
||||
document.getElementById('el-list').innerHTML='<div class="se-empty">请求失败: '+esc(e)+'</div>';
|
||||
var le=document.getElementById('el-list');
|
||||
if(le)le.innerHTML='<div class="se-empty">请求失败: '+esc(e)+'</div>';
|
||||
});
|
||||
},
|
||||
// 刷新截图
|
||||
@@ -1454,6 +1468,8 @@ var _stepEditor={
|
||||
if(infoEl)infoEl.textContent='失败';
|
||||
};
|
||||
img.src='/api/uiauto/screenshot?serial='+encodeURIComponent(serial)+'&_='+Date.now();
|
||||
// 同时重新拉取元素树——只刷截图会导致"截图是新的、元素框是旧的"错位
|
||||
this._loadElData();
|
||||
},
|
||||
// 在截图上渲染元素边界框（可点击）
|
||||
_renderOverlays:function(){
|
||||
@@ -1531,6 +1547,12 @@ var _stepEditor={
|
||||
item.classList.add('active');
|
||||
item.scrollIntoView({block:'nearest',behavior:'smooth'});
|
||||
}
|
||||
// 信息栏显示将填入的选择器，确认是否精确命中目标
|
||||
var infoEl=document.getElementById('el-shot-info');
|
||||
if(infoEl&&this._elData&&this._elData[i]){
|
||||
var sg=this._elData[i].suggested||{};
|
||||
infoEl.textContent='已选中: '+(sg.type||'')+' '+(sg.value||'');
|
||||
}
|
||||
},
|
||||
_renderEl:function(list){
|
||||
var self=this;
|
||||
@@ -1541,12 +1563,14 @@ var _stepEditor={
|
||||
if(el.text)attrs.push('<span>text:'+esc(el.text)+'</span>');
|
||||
if(el.description)attrs.push('<span>desc:'+esc(el.description)+'</span>');
|
||||
var indent=' '.repeat(Math.min(el.depth||0,6));
|
||||
return '<div class="el-picker-item" data-idx="'+idx+'" onclick="_stepEditor._pickEl('+idx+')">'+
|
||||
var sg=el.suggested||{};
|
||||
return '<div class="el-picker-item" data-idx="'+idx+'" title="点击填入: '+esc(sg.value||'')+'" onclick="_stepEditor._pickEl('+idx+')">'+
|
||||
'<span class="ep-depth">'+(el.depth||0)+'</span>'+
|
||||
'<div class="ep-info"><div class="ep-name">'+indent+esc(el.name||el.class||'')+'</div>'+
|
||||
'<div class="ep-attrs">'+attrs.join('')+'</div></div>'+
|
||||
'<span class="ep-sel">'+esc(el.suggested.type)+'</span>'+
|
||||
(el.suggested&&el.suggested.broad?' <span class="label label-warning" title="无唯一属性，选择器可能误匹配其他元素，建议选带文字/id 的元素">⚠宽泛</span>':'')+
|
||||
'<span class="ep-sel">'+esc(sg.type)+'</span>'+
|
||||
(sg.indexed?' <span class="label label-info" title="界面中同属性元素共 '+sg.total+' 个，按从上到下这是第 '+sg.occ+' 个，选择器已带序号精确定位">#'+sg.occ+'/'+sg.total+'</span>':'')+
|
||||
(sg.broad?' <span class="label label-warning" title="无唯一属性，选择器可能误匹配其他元素，建议选带文字/id 的元素">⚠宽泛</span>':'')+
|
||||
'</div>';
|
||||
}).join('');
|
||||
document.getElementById('el-list').innerHTML=html||'<div class="se-empty">无元素</div>';
|
||||
@@ -1834,17 +1858,20 @@ async function loadUsers(){
|
||||
filter:(u,kw)=>(u.username||'').toLowerCase().includes(kw),
|
||||
render:renderUserRow,
|
||||
tbody:'tb-users',pager:'pager-users',pageSize:10,
|
||||
empty:'<tr><td colspan="4" class="empty">暂无用户</td></tr>'
|
||||
empty:'<tr><td colspan="5" class="empty">暂无用户</td></tr>'
|
||||
});
|
||||
applyListPager('users');
|
||||
initListSearch('users','search-users');
|
||||
}
|
||||
function renderUserRow(u){
|
||||
const admin=u.is_admin?'<span class="label label-primary">管理员</span>':'<span class="label label-default">普通</span>';
|
||||
const perms=(u.perms||[]).map(p=>'<span class="label label-info">'+(PERM_LABELS[p]||p)+'</span>').join(' ')
|
||||
||'<span class="text-muted">-</span>';
|
||||
return '<tr>'+
|
||||
'<td>'+u.id+'</td>'+
|
||||
'<td><strong>'+esc(u.username)+'</strong></td>'+
|
||||
'<td>'+admin+'</td>'+
|
||||
'<td>'+perms+'</td>'+
|
||||
'<td>'+
|
||||
'<button class="btn btn-xs" onclick="openUserModal('+u.id+')">编辑</button>'+
|
||||
(u.username==='admin'?'':'<button class="btn btn-danger btn-xs" onclick="deleteUser('+u.id+')">删除</button>')+
|
||||
|
||||
+61
-6
@@ -596,18 +596,73 @@ def api_uiauto_status():
|
||||
return jsonify({"ok": True, "running": uiauto_helper.is_running()})
|
||||
|
||||
|
||||
@app.route("/api/steps/test", methods=["POST"])
|
||||
@perm_required(PERM_DEVICES)
|
||||
def api_steps_test():
|
||||
"""测试单个步骤：在指定设备上试执行，验证选择器是否命中（步骤编辑器"测试此步骤"）。
|
||||
|
||||
请求: {"serial": "100.100.10.11:5555", "step": {"type": "click", "params": {...}}}
|
||||
只读连接（adb connect + u2），不占用/释放 STF，与运行中任务互不干扰。
|
||||
返回 result: "命中" / "未找到" / "已执行"。
|
||||
"""
|
||||
data = request.json or {}
|
||||
serial = (data.get("serial") or "").strip()
|
||||
step = data.get("step") or {}
|
||||
if not serial:
|
||||
return jsonify({"ok": False, "error": "缺少 serial"}), 400
|
||||
if not step.get("type"):
|
||||
return jsonify({"ok": False, "error": "缺少步骤类型"}), 400
|
||||
from tasks.generic.task import test_step
|
||||
ok, msg, result = test_step(serial, step)
|
||||
res_txt = {True: "命中", False: "未找到"}.get(result, "已执行")
|
||||
_log.info(f"测试步骤 {step.get('type')} @ {serial}: {res_txt} ({msg})")
|
||||
return jsonify({"ok": ok, "msg": msg, "result": res_txt})
|
||||
|
||||
|
||||
@app.route("/api/uiauto/devices")
|
||||
@login_required
|
||||
@perm_required(PERM_DEVICES)
|
||||
def api_uiauto_devices():
|
||||
"""获取 uiauto2 已连接的设备列表（供抓取元素时选择设备）。"""
|
||||
"""获取可选设备列表（抓取元素用）。
|
||||
|
||||
uiautodev 只认识本地 adb 已连接的设备；平台设备池（STF）里在线但未
|
||||
本地连接的设备（如 100.100.10.10）在这里补全，并先做一次轻量 adb
|
||||
connect（单次尝试，绝不 kill-server），让它们可被 uiautodev 抓取。
|
||||
"""
|
||||
ok, data = uiauto_helper.list_devices()
|
||||
if ok:
|
||||
return jsonify({"ok": True, "devices": data})
|
||||
return jsonify({"ok": False, "error": data}), 503
|
||||
if not ok:
|
||||
return jsonify({"ok": False, "error": data}), 503
|
||||
seen = {d.get("serial") for d in data}
|
||||
try:
|
||||
pool = stf.list_all_devices()
|
||||
except Exception:
|
||||
pool = []
|
||||
added = 0
|
||||
for d in pool:
|
||||
serial = d.get("serial", "")
|
||||
if not serial or serial in seen:
|
||||
continue
|
||||
if not (d.get("present") and d.get("ready")):
|
||||
continue
|
||||
# 轻量连接（单次尝试，不重试不 kill-server）；连不上也照样列出，
|
||||
# 前端抓取时会报明确错误，不影响其他设备的连接
|
||||
adb_connect_light(serial)
|
||||
data.append({
|
||||
"serial": serial,
|
||||
"model": d.get("model") or d.get("product") or "",
|
||||
"product": d.get("product") or "",
|
||||
"name": serial,
|
||||
"status": "device",
|
||||
"enabled": True,
|
||||
})
|
||||
seen.add(serial)
|
||||
added += 1
|
||||
if added:
|
||||
_log.info(f"抓取元素设备列表补全 {added} 台 STF 在线设备: {list(seen)}")
|
||||
return jsonify({"ok": True, "devices": data})
|
||||
|
||||
|
||||
@app.route("/api/uiauto/screenshot")
|
||||
@login_required
|
||||
@perm_required(PERM_DEVICES)
|
||||
def api_uiauto_screenshot():
|
||||
"""通过 uiauto2 获取设备截图（供抓取元素时显示设备画面）。
|
||||
|
||||
|
||||
Reference in New Issue
Block a user