feat: 通用步骤新增动作(屏幕控制/按键/滑动直到元素/点击坐标/长按/等待元素/输入前清空)

This commit is contained in:
2026-08-11 08:27:33 +08:00
parent 3320035c9e
commit 003cb42cb1
2 changed files with 294 additions and 12 deletions
+57 -6
View File
@@ -744,7 +744,16 @@ function onTaskTypeChange(){
// ================== 步骤编辑器 ==================
var STEP_LIB=[
{type:'open_app',label:'打开App',icon:'▶',params:{package:'',wait_home:false,home_feature:''}},
{type:'stop_app',label:'结束App',icon:'⏹',params:{package:''}},
{type:'screen_on',label:'亮屏',icon:'💡',params:{}},
{type:'screen_off',label:'息屏',icon:'🌙',params:{}},
{type:'keep_screen',label:'保持亮屏',icon:'🔆',params:{mode:'on'}},
{type:'key_event',label:'按键',icon:'⌨️',params:{key:'back'}},
{type:'swipe',label:'滑动',icon:'↕',params:{direction:'up',duration_min:0.25,duration_max:0.50}},
{type:'swipe_until',label:'滑动直到元素',icon:'🔍',params:{selector_type:'xpath',selector_value:'',direction:'up',max_swipes:8,click_when_found:true}},
{type:'click_xy',label:'点击坐标',icon:'🎯',params:{x:50,y:50}},
{type:'long_click',label:'长按元素',icon:'👆',params:{selector_type:'xpath',selector_value:'',duration:1.0,wait_timeout:2}},
{type:'wait_el',label:'等待元素',icon:'⏳',params:{selector_type:'xpath',selector_value:'',timeout:10}},
{type:'click',label:'点击元素',icon:'✦',params:{selector_type:'xpath',selector_value:'',wait_timeout:2}},
{type:'input_text',label:'输入文字',icon:'⌨',params:{mode:'random',texts:'你好\n有趣\n支持',fixed_text:''}},
{type:'wait',label:'等待',icon:'⏱',params:{min:1.0,max:3.0}},
@@ -975,16 +984,48 @@ var _stepEditor={
h+='<div class="form-group"><label>包名</label><input type="text" class="form-control" data-key="package" value="'+esc(p.package||'')+'" placeholder="如 com.ss.android.ugc.aweme"></div>';
h+='<div class="form-row"><div class="form-group form-check"><input type="checkbox" data-key="wait_home" '+(p.wait_home?'checked':'')+'><label>等待进入首页</label></div>';
h+='<div class="form-group"><label>首页特征词</label><input type="text" class="form-control" data-key="home_feature" value="'+esc(p.home_feature||'')+'" placeholder="如 拍摄"></div></div>';
}else if(step.type==='stop_app'){
h+='<div class="form-group"><label>包名</label><input type="text" class="form-control" data-key="package" value="'+esc(p.package||'')+'" placeholder="如 com.ss.android.ugc.aweme"></div>';
h+='<div class="help">强制结束该 App(am force-stop),清除后台进程,下次打开为冷启动。常配合"打开App"使用:结束→重新打开</div>';
}else if(step.type==='screen_on'){
h+='<div class="help">亮屏(息屏时唤醒并滑动解锁)。任务执行前用——息屏状态下 u2 无法点击操作</div>';
}else if(step.type==='screen_off'){
h+='<div class="help">息屏。任务结束后用,避免长时间亮屏导致烧屏/发热</div>';
}else if(step.type==='keep_screen'){
h+='<div class="form-group"><label>模式</label><select class="form-control" data-key="mode">'+
'<option value="on"'+(p.mode!=='off'?' selected':'')+'>保持亮屏(充电时屏幕常亮)</option>'+
'<option value="off"'+(p.mode==='off'?' selected':'')+'>恢复自动息屏</option></select></div>';
h+='<div class="help">长任务(如养号看视频几小时)放"保持亮屏"防屏幕超时熄灭;任务结束前放"恢复自动息屏"</div>';
}else if(step.type==='swipe'){
h+='<div class="form-row"><div class="form-group"><label>方向</label><select class="form-control" data-key="direction">'+
['up','down','left','right'].map(function(d){return '<option value="'+d+'"'+(p.direction===d?' selected':'')+'>'+d+'</option>';}).join('')+'</select></div>';
h+='<div class="form-group"><label>最短时长(秒)</label><input type="number" class="form-control" data-key="duration_min" value="'+(p.duration_min||0.25)+'" step="0.05" min="0.05"></div>';
h+='<div class="form-group"><label>最长时长(秒)</label><input type="number" class="form-control" data-key="duration_max" value="'+(p.duration_max||0.50)+'" step="0.05" min="0.05"></div></div>';
}else if(step.type==='click'){
h+='<div class="form-row"><div class="form-group"><label>选择器类型</label><select class="form-control" data-key="selector_type">'+
['xpath','description','text','resourceId','descriptionContains','className'].map(function(d){return '<option value="'+d+'"'+(p.selector_type===d?' selected':'')+'>'+d+'</option>';}).join('')+'</select></div>';
h+='<div class="form-group" style="grid-column:span 2"><label>选择器值</label><div class="sel-picker"><input type="text" class="form-control" data-key="selector_value" value="'+esc(p.selector_value||'')+'" placeholder="如 点赞"><button class="btn btn-xs btn-pick" onclick="_stepEditor.openPicker(this)">抓取元素</button></div></div></div>';
h+='<div class="form-group"><label>等待超时(秒)</label><input type="number" class="form-control" data-key="wait_timeout" value="'+(p.wait_timeout||2)+'" step="0.5" min="0.5"></div>';
h+=_selRowHtml(p,'<div class="form-group"><label>等待超时(秒)</label><input type="number" class="form-control" data-key="wait_timeout" value="'+(p.wait_timeout||2)+'" step="0.5" min="0.5"></div>');
}else if(step.type==='long_click'){
h+=_selRowHtml(p,'<div class="form-row"><div class="form-group"><label>长按时长(秒)</label><input type="number" class="form-control" data-key="duration" value="'+(p.duration||1.0)+'" step="0.5" min="0.3"></div>'+
'<div class="form-group"><label>等待超时(秒)</label><input type="number" class="form-control" data-key="wait_timeout" value="'+(p.wait_timeout||2)+'" step="0.5" min="0.5"></div></div>');
}else if(step.type==='swipe_until'){
h+=_selRowHtml(p,'<div class="form-row"><div class="form-group"><label>方向</label><select class="form-control" data-key="direction">'+
'<option value="up"'+(p.direction!=='down'?' selected':'')+'>上滑</option>'+
'<option value="down"'+(p.direction==='down'?' selected':'')+'>下滑</option></select></div>'+
'<div class="form-group"><label>最多滑动次数</label><input type="number" class="form-control" data-key="max_swipes" value="'+(p.max_swipes||8)+'" min="1" max="50"></div></div>'+
'<div class="form-group form-check"><input type="checkbox" data-key="click_when_found" '+(p.click_when_found!==false?'checked':'')+'><label>找到后点击</label></div>'+
'<div class="help">上滑/下滑直到目标元素出现(每次滑动后检查一次),找不到则跳过继续</div>');
}else if(step.type==='wait_el'){
h+=_selRowHtml(p,'<div class="form-group"><label>最长等待(秒)</label><input type="number" class="form-control" data-key="timeout" value="'+(p.timeout||10)+'" min="1" max="120"></div>'+
'<div class="help">等待目标元素出现(替代固定秒数等待),超时则继续</div>');
}else if(step.type==='click_xy'){
h+='<div class="form-row"><div class="form-group"><label>X(屏幕% )</label><input type="number" class="form-control" data-key="x" value="'+(p.x||50)+'" min="0" max="100"></div>'+
'<div class="form-group"><label>Y(屏幕% )</label><input type="number" class="form-control" data-key="y" value="'+(p.y||50)+'" min="0" max="100"></div></div>'+
'<div class="help">屏幕百分比坐标,50/50=屏幕中心。无选择器时的兜底点击</div>';
}else if(step.type==='key_event'){
h+='<div class="form-group"><label>按键</label><select class="form-control" data-key="key">'+
[['back','返回键'],['home','Home键'],['enter','回车键'],['menu','菜单键'],['recent','最近任务'],['delete','删除键']].map(function(k){
return '<option value="'+k[0]+'"'+(p.key===k[0]?' selected':'')+'>'+k[1]+'</option>';
}).join('')+'</select></div>';
h+='<div class="help">返回键用于退出评论/返回上一页,比结束App轻量</div>';
}else if(step.type==='wait'){
h+='<div class="form-row"><div class="form-group"><label>最短(秒)</label><input type="number" class="form-control" data-key="min" value="'+(p.min||1.0)+'" step="0.5" min="0.1"></div>';
h+='<div class="form-group"><label>最长(秒)</label><input type="number" class="form-control" data-key="max" value="'+(p.max||3.0)+'" step="0.5" min="0.1"></div></div>';
@@ -995,6 +1036,7 @@ var _stepEditor={
'<option value="fixed"'+(im==='fixed'?' selected':'')+'>指定文字</option></select></div>';
h+='<div class="form-group" data-mode="random" style="display:'+(im==='random'?'block':'none')+'"><label>候选文字(每行一个,随机选一条)</label><textarea class="form-control" data-key="texts" rows="4" style="font-family:monospace;min-height:80px">'+esc(p.texts||'你好\n有趣\n支持')+'</textarea></div>';
h+='<div class="form-group" data-mode="fixed" style="display:'+(im==='fixed'?'block':'none')+'"><label>指定文字</label><input type="text" class="form-control" data-key="fixed_text" value="'+esc(p.fixed_text||'')+'"></div>';
h+='<div class="form-group form-check"><input type="checkbox" data-key="clear_first" '+(p.clear_first!==false?'checked':'')+'><label>输入前先清空</label></div>';
h+='<div class="help">需先用"点击元素"步骤定位到输入框,本步骤只负责输入文字</div>';
}else if(step.type==='loop'){
const lm=p.loop_mode==='time'?'time':'rounds';
@@ -1295,9 +1337,18 @@ var _stepEditor={
if(!s.params.selector_value){
warnings.push(name+' "'+label+'": 选择器值为空,点击步骤会跳过。请用"抓取元素"按钮选择目标。');
}
}else if(s.type==='open_app'){
}else if(s.type==='long_click'||s.type==='swipe_until'||s.type==='wait_el'){
if(!s.params.selector_value){
warnings.push(name+' "'+label+'": 选择器值为空,该步骤会跳过。请用"抓取元素"或"测试"按钮选择目标。');
}
}else if(s.type==='click_xy'){
const x=parseFloat(s.params.x), y=parseFloat(s.params.y);
if(isNaN(x)||isNaN(y)||x<0||x>100||y<0||y>100){
warnings.push(name+' "'+label+'": 坐标需在 0-100 之间(屏幕百分比)。');
}
}else if(s.type==='open_app'||s.type==='stop_app'){
if(!s.params.package){
warnings.push(name+' "'+label+'": 包名为空,无法启动App');
warnings.push(name+' "'+label+'": 包名为空,无法'+(s.type==='open_app'?'启动':'结束')+'App');
}
}else if(s.type==='input_text'){
if(s.params.mode==='fixed'&&!s.params.fixed_text){
+237 -6
View File
@@ -13,9 +13,18 @@ worker 按 steps 顺序执行,支持 loop 步骤循环、停止信号、进度
支持的步骤类型:
open_app - 启动 app
stop_app - 强制结束 app(am force-stop,下次打开冷启动)
screen_on - 亮屏(息屏时唤醒并滑动解锁)
screen_off - 息屏
keep_screen - 保持亮屏/恢复自动息屏(svc power stayon,设备充电时常亮)
key_event - 按键(返回/Home/回车/菜单等,d.press)
swipe - 滑动(方向/时长)
swipe_until - 滑动直到元素出现(最多 N 次,可选中找到后点击)
click - 点击元素(选择器)
input_text - 输入文字(随机候选/指定文字)
click_xy - 点击坐标(屏幕百分比,无选择器时兜底)
long_click - 长按元素(选择器 + 时长)
wait_el - 等待元素出现(条件等待,替代固定时长 wait)
input_text - 输入文字(随机候选/指定文字,可先清空)
wait - 等待时长
loop - 循环块(含 children 步骤列表 + max_iterations)
group - 动作组(含 children 步骤列表,按序执行一次,可折叠复用)
@@ -35,12 +44,31 @@ _log = get_logger("task.generic")
STEP_TYPES = [
{"type": "open_app", "label": "打开App", "icon": "▶",
"params": {"package": "", "wait_home": False, "home_feature": ""}},
{"type": "stop_app", "label": "结束App", "icon": "⏹",
"params": {"package": ""}},
{"type": "screen_on", "label": "亮屏", "icon": "💡",
"params": {}},
{"type": "screen_off", "label": "息屏", "icon": "🌙",
"params": {}},
{"type": "keep_screen", "label": "保持亮屏", "icon": "🔆",
"params": {"mode": "on"}},
{"type": "key_event", "label": "按键", "icon": "⌨️",
"params": {"key": "back"}},
{"type": "swipe", "label": "滑动", "icon": "↕",
"params": {"direction": "up", "duration_min": 0.25, "duration_max": 0.50}},
{"type": "swipe_until", "label": "滑动直到元素", "icon": "🔍",
"params": {"selector_type": "xpath", "selector_value": "", "direction": "up",
"max_swipes": 8, "click_when_found": True}},
{"type": "click", "label": "点击元素", "icon": "✦",
"params": {"selector_type": "description", "selector_value": "", "wait_timeout": 2}},
{"type": "click_xy", "label": "点击坐标", "icon": "🎯",
"params": {"x": 50, "y": 50}},
{"type": "long_click", "label": "长按元素", "icon": "👆",
"params": {"selector_type": "xpath", "selector_value": "", "duration": 1.0, "wait_timeout": 2}},
{"type": "wait_el", "label": "等待元素", "icon": "⏳",
"params": {"selector_type": "xpath", "selector_value": "", "timeout": 10}},
{"type": "input_text", "label": "输入文字", "icon": "⌨",
"params": {"mode": "random", "texts": "你好\n有趣\n支持", "fixed_text": ""}},
"params": {"mode": "random", "texts": "你好\n有趣\n支持", "fixed_text": "", "clear_first": True}},
{"type": "wait", "label": "等待", "icon": "⏱",
"params": {"min": 1.0, "max": 3.0}},
{"type": "loop", "label": "循环块", "icon": "↻",
@@ -140,10 +168,11 @@ class GenericStepsWorker(BaseWorker):
self.set_action(f"执行: {label}")
_log.info(f"[{self.serial}] 步骤: {label}({stype}) params={params}")
ret = None
handler = getattr(self, f"_exec_{stype}", None)
if handler:
try:
handler(d, params, depth)
ret = handler(d, params, depth)
except Exception as e:
_log.error(f"[{self.serial}] 步骤 {label}({stype}) 异常: {e}")
else:
@@ -156,8 +185,54 @@ class GenericStepsWorker(BaseWorker):
self.set_progress(done=sum(self._action_counts.values()),
total=0, unit="操作",
action_counts=dict(self._action_counts), elapsed=elapsed)
return ret # 命中结果(True/False/None),"测试此步骤"用
# ================== 步骤执行器 ==================
def _exec_screen_on(self, d, params, depth=0):
"""亮屏:息屏时唤醒并滑动解锁。任务执行前用(息屏时 u2 无法操作)。"""
try:
d.unlock()
_log.info(f"[{self.serial}] 亮屏")
except Exception as e:
_log.warning(f"[{self.serial}] 亮屏异常: {e}")
def _exec_screen_off(self, d, params, depth=0):
"""息屏:任务结束后用,避免长时间亮屏(烧屏/发热)。"""
try:
d.screen_off()
_log.info(f"[{self.serial}] 息屏")
except Exception as e:
_log.warning(f"[{self.serial}] 息屏异常: {e}")
def _exec_keep_screen(self, d, params, depth=0):
"""保持亮屏 / 恢复自动息屏。
长任务(养号看视频数小时)时屏幕会按系统超时自动熄灭导致任务无法执行,
用 svc power stayon 让充电时屏幕常亮;任务结束前用 mode=off 恢复自动息屏。
"""
mode = params.get("mode", "on")
try:
if mode == "off":
d.shell("svc power stayon false")
_log.info(f"[{self.serial}] 恢复自动息屏")
else:
d.shell("svc power stayon true")
_log.info(f"[{self.serial}] 保持亮屏(充电时屏幕常亮)")
except Exception as e:
_log.warning(f"[{self.serial}] keep_screen({mode}) 异常: {e}")
def _exec_stop_app(self, d, params, depth=0):
"""强制结束 App(am force-stop)。清后台不留进程,下次打开为冷启动。"""
package = params.get("package", "")
if not package:
_log.warning(f"[{self.serial}] stop_app 缺少 package 参数")
return
try:
d.app_stop(package)
_log.info(f"[{self.serial}] 已强制结束 {package}")
except Exception as e:
_log.warning(f"[{self.serial}] 结束 {package} 异常: {e}")
def _exec_open_app(self, d, params, depth=0):
package = params.get("package", "")
if not package:
@@ -171,6 +246,125 @@ class GenericStepsWorker(BaseWorker):
if not wait_for_app_home(d, package, home_check, timeout=40):
self.set_action(f"{package} 首页加载超时,继续")
def _exec_key_event(self, d, params, depth=0):
"""按键:返回/Home/回车等(d.press)。退出评论、返回上一页必备。"""
key = params.get("key", "back")
try:
d.press(key)
_log.info(f"[{self.serial}] 按键: {key}")
except Exception as e:
_log.warning(f"[{self.serial}] 按键 {key} 异常: {e}")
def _exec_swipe_until(self, d, params, depth=0):
"""滑动直到元素出现(最多 max_swipes 次),可选找到后点击。
养号核心动作:信息流刷到目标按钮(点赞/评论)再操作。
返回 True=找到, False=未找到, None=配置缺选择器。
"""
sel_type = params.get("selector_type", "xpath")
sel_val = params.get("selector_value", "")
direction = params.get("direction", "up")
max_swipes = max(1, int(params.get("max_swipes", 8)))
click_when_found = params.get("click_when_found", True)
if not sel_val:
_log.warning("swipe_until 缺少选择器")
return None
info = d.info
w, h = info["displayWidth"], info["displayHeight"]
cx = int(w * 0.5)
if direction == "down":
sy, ey = int(h * 0.2), int(h * 0.8)
else:
sy, ey = int(h * 0.8), int(h * 0.2)
for i in range(1, max_swipes + 1):
if self.stopped() or self.is_time_up():
return None
found = False
try:
if sel_type == "xpath":
el = d.xpath(sel_val)
found = el.wait(timeout=1)
else:
el = d(**{sel_type: sel_val})
found = el.exists(timeout=1)
except Exception:
found = False
if found:
self.set_action(f"滑动找到元素(第{i}次)")
if click_when_found:
try:
el.click()
_log.info(f"[{self.serial}] 滑动{i}次后找到并点击: {sel_val}")
except Exception as e:
_log.warning(f"[{self.serial}] 找到但点击失败: {e}")
else:
_log.info(f"[{self.serial}] 滑动{i}次后找到: {sel_val}")
self._track_selector_health(sel_val, True)
return True
d.swipe(cx, sy, cx, ey, 0.3)
time.sleep(0.5)
self._track_selector_health(sel_val, False)
_log.warning(f"[{self.serial}] 滑动{max_swipes}次未找到元素: {sel_val}")
return False
def _exec_click_xy(self, d, params, depth=0):
"""点击坐标(屏幕百分比 0-100,50/50=屏幕中心)。无选择器时兜底。"""
x_pct = float(params.get("x", 50))
y_pct = float(params.get("y", 50))
info = d.info
w, h = info["displayWidth"], info["displayHeight"]
x = int(w * min(max(x_pct, 0), 100) / 100)
y = int(h * min(max(y_pct, 0), 100) / 100)
d.click(x, y)
_log.info(f"[{self.serial}] 点击坐标 ({x},{y})")
def _exec_long_click(self, d, params, depth=0):
"""长按元素(选择器 + 时长秒)。复制链接/呼出菜单用。"""
sel_type = params.get("selector_type", "xpath")
sel_val = params.get("selector_value", "")
duration = float(params.get("duration", 1.0))
timeout = float(params.get("wait_timeout", 2))
if not sel_val:
_log.warning("long_click 缺少选择器")
return None
found = False
try:
if sel_type == "xpath":
el = d.xpath(sel_val)
if el.wait(timeout=timeout):
el.long_click(duration=duration)
found = True
else:
el = d(**{sel_type: sel_val})
if el.exists(timeout=timeout):
el.long_click(duration=duration)
found = True
self._track_selector_health(sel_val, found)
_log.info(f"[{self.serial}] 长按 {'命中' if found else '未找到'}: {sel_val}")
except Exception as e:
_log.warning(f"[{self.serial}] 长按异常: {e}")
return found
def _exec_wait_el(self, d, params, depth=0):
"""等待元素出现(条件等待)。返回 True=出现, False=超时, None=缺选择器。"""
sel_type = params.get("selector_type", "xpath")
sel_val = params.get("selector_value", "")
timeout = float(params.get("timeout", 10))
if not sel_val:
_log.warning("wait_el 缺少选择器")
return None
try:
if sel_type == "xpath":
found = d.xpath(sel_val).wait(timeout=timeout)
else:
found = d(**{sel_type: sel_val}).exists(timeout=timeout)
self._track_selector_health(sel_val, found)
_log.info(f"[{self.serial}] 等待元素 {'出现' if found else '超时未出现'}: {sel_val}")
return found
except Exception as e:
_log.warning(f"[{self.serial}] 等待元素异常: {e}")
return False
def _exec_swipe(self, d, params, depth=0):
direction = params.get("direction", "up")
dur_min = float(params.get("duration_min", 0.25))
@@ -221,6 +415,7 @@ class GenericStepsWorker(BaseWorker):
self._track_selector_health(sel_val, found)
except Exception as e:
_log.warning(f"[{self.serial}] click 异常: {e}")
return found # 命中结果,"测试此步骤"用
def _exec_wait(self, d, params, depth=0):
min_s = float(params.get("min", 1.0))
@@ -284,15 +479,18 @@ class GenericStepsWorker(BaseWorker):
text = random.choice(candidates)
if not text:
return
clear_first = params.get("clear_first", True)
try:
# 优先用 u2 的 send_keys(对当前焦点元素输入)
d.send_keys(text)
_log.info(f"[{self.serial}] input_text: {text}")
# 优先用 u2 的 send_keys(对当前焦点元素输入);clear=True 先清空再输入
d.send_keys(text, clear=clear_first)
_log.info(f"[{self.serial}] input_text: {text}" + ("(已先清空)" if clear_first else ""))
except Exception as e:
# 兜底:找 EditText 设置文本
try:
edit = d(className="android.widget.EditText")
if edit.exists:
if clear_first:
edit.clear_text()
edit.set_text(text)
_log.info(f"[{self.serial}] input_text(set_text): {text}")
else:
@@ -321,3 +519,36 @@ class GenericStepsTask(BaseTask):
def create_worker(self, stf, serial, params):
merged = {**DEFAULT_PARAMS, **(params or {})}
return GenericStepsWorker(stf, serial, params=merged)
def test_step(serial, step):
"""在设备上单步试执行(编辑器"测试此步骤"按钮用)。
直接 adb connect + u2 连接后执行单个步骤,验证选择器是否命中。
返回 (ok, msg, result):result 为执行器返回的命中结果(True/False/None)。
与运行中的任务互不干扰(只读连接,不占用/不释放 STF)。
"""
import uiautomator2 as u2
from concurrent.futures import ThreadPoolExecutor # 与 device_worker 一致(本机 stdlib threading 无此属性)
from core.adb_helper import adb_connect
from core.device_worker import _U2_CONNECT_TIMEOUT
try:
if not adb_connect(serial):
return False, f"adb connect {serial} 失败", None
except Exception as e:
return False, f"adb connect 异常: {e}", None
try:
with ThreadPoolExecutor(max_workers=1) as pool:
fut = pool.submit(u2.connect, serial)
d = fut.result(timeout=_U2_CONNECT_TIMEOUT)
except Exception as e:
return False, f"u2 连接失败: {e}", None
if d is None:
return False, "u2 连接超时", None
try:
w = GenericStepsWorker(None, serial, {})
result = w._exec_one(d, step)
return True, f"步骤已执行({step.get('label') or step.get('type')})", result
except Exception as e:
return False, f"执行异常: {e}", None