Files
auto_control/web/tasks_api.py
T
butubb 73895f7233 feat(任务): 录制手势改成**纯录制回放**(真手指轨迹)+ 手机端 getevent 录制
用户反馈:录制出来的滑动又被套上滑动那套逻辑(方向/幅度/拟人重新生成),
"导致滑动还是不顺畅"。录制就该是录制——按录下的路径与时间原样重放。

- 新步骤 `gesture`「录制手势」:存完整轨迹点列 `[[x,y,t_ms],…]`,
  回放时原样交给设备(`d.swipe_points(points, duration)`),不做任何加工。
  与 `swipe` 是两套东西(滑动是参数化的,录制是点列)。
- core/gesture.py(新):
  · 手机端录制 `PhoneRecorder`:`getevent` 读**真触屏**设备(自动挑 fts_ts 这类、
    排除 uinput/vitural-sar 等合成节点),解析两套协议(BTN_TOUCH / ABS_MT_TRACKING_ID);
    **合成的注入事件不会出现在真触屏节点上**(实测),所以录到的只有人手的动作;
  · 点列清洗:按 ≥16ms 抽稀但**末点必留**(快划时不能把收尾丢了);
  · 回放**一次调用**而不是逐点注入:实测这台设备单次触摸 RPC ≈190ms,
    逐点回放 20 点要 3.8 秒——只能让设备自己插值,"时间"由点密度还原。
- 接口:POST /api/gesture/record/{start,stop}(需设备权限;重复开始返回 409)。
- 「录制手势」步骤卡片:录到就回填(手机上录 / 网页上录),显示点数/时长;
  **滑动步骤上的「录制手势」按钮已移除**(按用户要求,录制不再走滑动逻辑)。
- 录制前自动唤醒设备:**息屏时 u2 抓 UI 树会从 3 秒退化到 65 秒**(实测),
  截图也是黑的——这是本次排查最耗时的坑,已写进文档。

自测:假设备单测(点列/时长/抽稀/失败退回直线,10 项);
无头浏览器 + CDP 真模拟拖动 → 录到 14 点/605ms 并按实际轨迹回填;
手机录制接口起停/重复拦截;
真机回放:用录下的轨迹跑任务 → 步骤 `gesture ok`。
⚠️ 手机端"真手指轨迹"这一段需要人真的去划才能端到端验证(我没有手指)。
文档:TASK_DEV §3.2(两套东西对比 + 三个坑)、步骤表 21 种、API、README。
2026-09-20 15:04:21 +08:00

520 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""任务域 API:任务计划/分组/自定义动作/元素抓取/步骤测试。"""
import re
import time
import uuid
import threading
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError as FuturesTimeout
from flask import (Blueprint, jsonify, request, Response)
from flask_login import login_required, current_user
from core import device_pool, uiauto_helper, step_defaults
from core.adb_helper import adb_connect_light
from core.models import db, DeviceGroup, TaskJob, CustomAction
from core.logger import get_logger
_log = get_logger("web")
from web import context
from web.auth import perm_required, PERM_DEVICES, PERM_TASKS
from tasks import list_task_types, get_task_class
from web.common import _merged_device_list
bp = Blueprint("tasks", __name__)
@bp.route("/api/task_types")
@login_required
def api_task_types():
return jsonify({"ok": True, "task_types": list_task_types()})
@bp.route("/api/actions")
@login_required
def api_actions():
"""返回指定任务类型支持的专属操作。"""
task_type = request.args.get("task_type", "")
task_cls = get_task_class(task_type)
if not task_cls:
return jsonify({"ok": False, "error": "未知任务类型", "actions": []})
actions = task_cls.list_action_types()
return jsonify({"ok": True, "actions": actions})
@bp.route("/api/devices")
@login_required
def api_devices():
"""在线设备列表(供分组表单/任务目标/AI 的「选择设备」下拉用)。
- `devices`:serial 字符串列表(兼容既有调用方)
- `items`:`[{serial, name, model}]` —— 界面显示**设备名称**用
(名称是设备的身份,见 doc/API.md §6)
"""
try:
serials = context.mgr.list_all_serials()
info = {}
try:
for d in device_pool.list_devices():
info[d["serial"]] = d
except Exception:
pass
items = [{"serial": s,
"name": (info.get(s) or {}).get("name") or "",
"model": (info.get(s) or {}).get("model") or ""} for s in serials]
return jsonify({"ok": True, "devices": serials, "items": items})
except Exception as e:
return jsonify({"ok": False, "error": str(e)}), 500
# ================== API:任务计划 CRUD ==================
def _job_next_run(job):
"""任务下次执行时间,格式化为 "YYYY-MM-DD HH:MM"(Flask 序列化 aware datetime 会变 GMT 格式)。"""
nr = context.mgr.next_run_of(job)
return nr.strftime("%Y-%m-%d %H:%M") if nr else None
def _job_coverage(job, configured):
"""任务覆盖的设备(监控页「任务运行概况」展示用)。
与 TaskJob.resolve_serials 的差别:这里按 **target 定义**给出「这台任务会覆盖
哪些设备」,不因设备当前是否空闲而变(在线/运行中由前端拿实时状态标注),
也不做离线过滤、不写调度日志——它是每 5s 轮询的展示接口。
"""
tgt = job.target or {}
mode = tgt.get("mode", "all")
try:
if mode == "serial":
serials = [tgt.get("serial") or ""]
elif mode == "group":
g = context.mgr.groups.get(tgt.get("group_name"))
serials = [s for s in (list(g.serials) if g else []) if s in configured]
else:
serials = sorted(configured)
serials = [s for s in serials if s]
return {"mode": mode, "serials": serials, "total": len(serials)}
except Exception as e:
_log.warning(f"任务「{job.name}」覆盖设备解析失败: {e}")
return {"mode": mode, "serials": [], "total": 0, "error": str(e)[:80]}
def _configured_set():
"""设备池清单(coverage 计算用,取一次给所有任务复用)。"""
try:
return set(device_pool.list_configured())
except Exception:
return set()
@bp.route("/api/jobs")
@login_required
def api_jobs_list():
configured = _configured_set()
jobs = []
for j in context.mgr.jobs.values():
d = j.to_dict()
d["next_run"] = _job_next_run(j)
d["coverage"] = _job_coverage(j, configured)
jobs.append(d)
return jsonify({"ok": True, "jobs": jobs,
"task_types": list_task_types()})
@bp.route("/api/jobs", methods=["POST"])
@perm_required(PERM_TASKS)
def api_jobs_create():
data = request.json or {}
name = (data.get("name") or "").strip()
if not name:
return jsonify({"ok": False, "error": "任务名不能为空"}), 400
task_type = data.get("task_type", "generic_steps")
if not get_task_class(task_type):
return jsonify({"ok": False, "error": f"未知任务类型: {task_type}"}), 400
job = context.mgr.add_job(
name=name, task_type=task_type,
target=data.get("target", {"mode": "all"}),
params=data.get("params", {}),
schedule=data.get("schedule", {"mode": "once"}),
retry=data.get("retry", {"max_attempts": 1, "delay": 60}),
enabled=data.get("enabled", True))
d = job.to_dict()
d["next_run"] = _job_next_run(job)
d["coverage"] = _job_coverage(job, _configured_set())
return jsonify({"ok": True, "msg": "任务已创建", "job": d})
@bp.route("/api/jobs/<job_id>", methods=["PUT"])
@perm_required(PERM_TASKS)
def api_jobs_update(job_id):
job = context.mgr.jobs.get(job_id)
if not job:
return jsonify({"ok": False, "error": "任务不存在"}), 404
data = request.json or {}
fields = {}
for k in ("name", "task_type", "target", "params", "schedule", "retry", "enabled"):
if k in data:
fields[k] = data[k]
if "task_type" in fields and not get_task_class(fields["task_type"]):
return jsonify({"ok": False, "error": f"未知任务类型: {fields['task_type']}"}), 400
context.mgr.update_job(job_id, **fields)
d = job.to_dict()
d["next_run"] = _job_next_run(job)
d["coverage"] = _job_coverage(job, _configured_set())
return jsonify({"ok": True, "msg": "任务已更新", "job": d})
@bp.route("/api/jobs/<job_id>", methods=["DELETE"])
@perm_required(PERM_TASKS)
def api_jobs_delete(job_id):
if context.mgr.delete_job(job_id):
return jsonify({"ok": True, "msg": "任务已删除"})
return jsonify({"ok": False, "error": "任务不存在"}), 404
@bp.route("/api/jobs/<job_id>/run", methods=["POST"])
@perm_required(PERM_TASKS)
def api_jobs_run(job_id):
return jsonify(context.mgr.run_job_now(job_id))
@bp.route("/api/jobs/<job_id>/toggle", methods=["POST"])
@perm_required(PERM_TASKS)
def api_jobs_toggle(job_id):
enabled = (request.json or {}).get("enabled", True)
job = context.mgr.toggle_job(job_id, enabled)
if not job:
return jsonify({"ok": False, "error": "任务不存在"}), 404
return jsonify({"ok": True, "msg": f"任务已{'启用' if enabled else '停用'}"})
# ================== API:设备分组 CRUD ==================
@bp.route("/api/groups")
@login_required
def api_groups_list():
return jsonify({"ok": True, "groups": [g.to_dict() for g in context.mgr.groups.values()]})
@bp.route("/api/groups", methods=["POST"])
@perm_required(PERM_TASKS)
def api_groups_create():
data = request.json or {}
name = (data.get("name") or "").strip()
if not name:
return jsonify({"ok": False, "error": "分组名不能为空"}), 400
if name in context.mgr.groups:
return jsonify({"ok": False, "error": "分组名已存在"}), 400
context.mgr.add_group(name, data.get("serials", []), data.get("description", ""))
return jsonify({"ok": True, "msg": "分组已创建"})
@bp.route("/api/groups/<name>", methods=["PUT"])
@perm_required(PERM_TASKS)
def api_groups_update(name):
if name not in context.mgr.groups:
return jsonify({"ok": False, "error": "分组不存在"}), 404
data = request.json or {}
context.mgr.update_group(name,
serials=data.get("serials"),
description=data.get("description"))
return jsonify({"ok": True, "msg": "分组已更新"})
@bp.route("/api/groups/<name>", methods=["DELETE"])
@perm_required(PERM_TASKS)
def api_groups_delete(name):
if context.mgr.delete_group(name):
return jsonify({"ok": True, "msg": "分组已删除"})
return jsonify({"ok": False, "error": "分组不存在"}), 404
# ================== API:录制手势(手机端真手指轨迹) ==================
#
# 用 getevent 读**真触屏**设备抓手指轨迹(见 core/gesture.py)。合成的注入事件不会
# 出现在真触屏节点上,所以录到的只有人手的动作。
# 录制器实例存在内存里(单进程;不落库——录完的点列由前端放进步骤/动作再保存)。
_gesture_recorders = {} # token -> PhoneRecorder
_gesture_lock = threading.Lock()
_GESTURE_TTL = 300 # 超过 5 分钟自动收摊,避免忘了点"停止"
def _gesture_sweep():
"""清掉超时的录制器(停止并释放 getevent 进程)。"""
now = time.time()
with _gesture_lock:
stale = [t for t, r in _gesture_recorders.items()
if now - r._started > _GESTURE_TTL]
for t in stale:
_gesture_recorders.pop(t, None)
for t in stale:
try:
r.stop()
except Exception:
pass
_log.info("录制手势超时自动停止: %s", t)
@bp.route("/api/gesture/record/start", methods=["POST"])
@perm_required(PERM_DEVICES)
def api_gesture_record_start():
"""开始录:{serial} → {token}。之后在手机屏幕上用手指划,停止时取回轨迹。"""
from core import gesture
_gesture_sweep()
serial = ((request.json or {}).get("serial") or "").strip()
if not serial:
return jsonify({"ok": False, "error": "缺少 serial"}), 400
with _gesture_lock:
if _gesture_recorders:
return jsonify({"ok": False, "error": "已有录制在进行中,先停止"}), 409
try:
w, h = 0, 0
try:
import uiautomator2 as u2
from core.u2_helper import screen_size
d = u2.connect(serial)
d.unlock() # 息屏时 u2 抓树会退化到几十秒,先唤醒
w, h = screen_size(d, serial)
except Exception as e:
_log.warning("录制前取屏幕尺寸失败(按原始坐标范围换算): %s", e)
rec = gesture.PhoneRecorder(serial, w, h)
rec.start()
except Exception as e:
return jsonify({"ok": False, "error": f"开始录制失败: {e}"}), 502
token = uuid.uuid4().hex[:12]
with _gesture_lock:
_gesture_recorders[token] = rec
_log.info("开始录制手势: %s @ %s(%s)by %s", token, serial, rec.device,
getattr(current_user, "username", ""))
return jsonify({"ok": True, "token": token, "device": rec.device,
"screen": [w, h],
"msg": "现在请用手指在手机上划,划完点「停止」"})
@bp.route("/api/gesture/record/stop", methods=["POST"])
@perm_required(PERM_DEVICES)
def api_gesture_record_stop():
"""停止录制:{token} → {gestures: [[[x,y,t_ms],...], ...]}。"""
token = ((request.json or {}).get("token") or "").strip()
with _gesture_lock:
rec = _gesture_recorders.pop(token, None)
if rec is None:
return jsonify({"ok": False, "error": "录制会话不存在或已超时"}), 404
try:
gestures = rec.stop()
except Exception as e:
return jsonify({"ok": False, "error": f"停止录制失败: {e}"}), 500
_log.info("停止录制手势: %s → %s 条", token, len(gestures))
return jsonify({"ok": True, "gestures": gestures, "count": len(gestures),
"device": rec.device})
# ================== API:步骤默认值(「动作配置」页) ==================
#
# 存 app_meta.step_defaults 一个键(不建表),校验与合并都在 core/step_defaults.py。
@bp.route("/api/step_defaults")
@login_required
def api_step_defaults_get():
"""生效的默认值 + 出厂值 + 字段规格(前端画表单用)。"""
return jsonify({"ok": True, **step_defaults.as_payload()})
@bp.route("/api/step_defaults", methods=["POST"])
@perm_required(PERM_TASKS)
def api_step_defaults_save():
"""保存默认值(只落与出厂值不同的字段,便于以后出厂值升级能跟着走)。"""
data = request.json or {}
ok, msg = step_defaults.save(data.get("defaults") or data)
if not ok:
return jsonify({"ok": False, "error": msg}), 400
_log.info("更新步骤默认值 by %s: %s", getattr(current_user, "username", ""),
list((data.get("defaults") or data).keys()))
return jsonify({"ok": True, "msg": msg, **step_defaults.as_payload()})
# ================== API:用户管理 CRUD ==================
@bp.route("/api/custom_actions")
@login_required
def api_custom_actions_list():
rows = CustomAction.query.order_by(CustomAction.created_at.desc()).all()
return jsonify({"ok": True, "actions": [r.to_dict() for r in rows]})
@bp.route("/api/custom_actions", methods=["POST"])
@perm_required(PERM_TASKS)
def api_custom_actions_create():
data = request.json or {}
name = (data.get("name") or "").strip()
if not name:
return jsonify({"ok": False, "error": "动作名不能为空"}), 400
steps = data.get("steps", [])
if not steps:
return jsonify({"ok": False, "error": "至少需要1个步骤"}), 400
row = CustomAction(id=uuid.uuid4().hex[:8], name=name,
icon=data.get("icon") or "📦",
created_at=datetime.now().strftime("%Y-%m-%d %H:%M"))
row.set_steps(steps)
db.session.add(row)
db.session.commit()
return jsonify({"ok": True, "msg": "动作已保存", "action": row.to_dict()})
@bp.route("/api/custom_actions/<action_id>", methods=["PUT"])
@perm_required(PERM_TASKS)
def api_custom_actions_update(action_id):
row = CustomAction.query.get(action_id)
if not row:
return jsonify({"ok": False, "error": "动作不存在"}), 404
data = request.json or {}
if "name" in data:
name = (data["name"] or "").strip()
if not name:
return jsonify({"ok": False, "error": "动作名不能为空"}), 400
row.name = name
if "icon" in data:
row.icon = data["icon"] or "📦"
if "steps" in data:
if not data["steps"]:
return jsonify({"ok": False, "error": "至少需要1个步骤"}), 400
row.set_steps(data["steps"])
db.session.commit()
return jsonify({"ok": True, "msg": "已更新", "action": row.to_dict()})
@bp.route("/api/custom_actions/<action_id>", methods=["DELETE"])
@perm_required(PERM_TASKS)
def api_custom_actions_delete(action_id):
row = CustomAction.query.get(action_id)
if not row:
return jsonify({"ok": False, "error": "动作不存在"}), 404
db.session.delete(row)
db.session.commit()
return jsonify({"ok": True, "msg": "已删除"})
@bp.route("/api/uiauto/elements")
@perm_required(PERM_DEVICES)
def api_uiauto_elements():
"""获取设备当前 UI 元素树(供步骤编辑器"抓取元素"用)。
依赖本地运行的 uiautodev 服务(端口 20242)。
?serial=xxx 设备 serial
返回:
200 — {ok:true, elements:[...]}
503 — {ok:false, error:"..."}(uiauto2 未启动)
"""
serial = request.args.get("serial", "")
if not serial:
return jsonify({"ok": False, "error": "缺少 serial"}), 400
ok, data = uiauto_helper.get_elements(serial)
if ok:
return jsonify({"ok": True, "elements": data})
return jsonify({"ok": False, "error": data}), 503
@bp.route("/api/uiauto/status")
@login_required
def api_uiauto_status():
"""探测 uiauto2 本地服务是否运行(前端按钮禁启用)。"""
return jsonify({"ok": True, "running": uiauto_helper.is_running()})
@bp.route("/api/steps/test", methods=["POST"])
@perm_required(PERM_DEVICES)
def api_steps_test():
"""测试单个步骤:在指定设备上试执行,验证选择器是否命中(步骤编辑器"测试此步骤")。
请求: {"serial": "100.100.10.11:5555", "step": {"type": "click", "params": {...}}}
只读连接(adb connect + u2),与运行中任务互不干扰。
返回 result: "命中" / "未找到" / "已执行"。
"""
data = request.json or {}
serial = (data.get("serial") or "").strip()
step = data.get("step") or {}
if not serial:
return jsonify({"ok": False, "error": "缺少 serial"}), 400
if not step.get("type"):
return jsonify({"ok": False, "error": "缺少步骤类型"}), 400
from tasks.generic.task import test_step
ok, msg, result = test_step(serial, step)
res_txt = {True: "命中", False: "未找到"}.get(result, "已执行")
_log.info(f"测试步骤 {step.get('type')} @ {serial}: {res_txt} ({msg})")
return jsonify({"ok": ok, "msg": msg, "result": res_txt})
@bp.route("/api/uiauto/devices")
@perm_required(PERM_DEVICES)
def api_uiauto_devices():
"""获取可选设备列表(抓取元素用)。
uiautodev 只认识本地 adb 已连接的设备;设备池里在线但未本地连接的设备
在这里补全,并先做一次轻量 adb connect(单次尝试,绝不 kill-server),
让它们可被 uiautodev 抓取。
"""
ok, data = uiauto_helper.list_devices()
if not ok:
return jsonify({"ok": False, "error": data}), 503
seen = {d.get("serial") for d in data}
try:
online = set(device_pool.list_online())
pool = [s for s in device_pool.list_configured() if s in online]
except Exception:
pool = []
added = 0
for serial in pool:
if not serial or serial in seen:
continue
# 轻量连接(单次尝试,不重试不 kill-server);连不上也照样列出,
# 前端抓取时会报明确错误,不影响其他设备的连接
adb_connect_light(serial)
data.append({
"serial": serial,
"model": "",
"product": "",
"name": serial,
"status": "device",
"enabled": True,
})
seen.add(serial)
added += 1
if added:
_log.info(f"抓取元素设备列表补全 {added} 台在线设备: {list(seen)}")
return jsonify({"ok": True, "devices": data})
@bp.route("/api/uiauto/snapshot")
@perm_required(PERM_DEVICES)
def api_uiauto_snapshot():
"""一次取齐:设备截图 + 元素树(**原生 u2,同一个连接背靠背取**)。
抓取元素时用它代替「/api/uiauto/screenshot + /api/uiauto/elements」两连击:
那两条是两次独立取数,中间隔着 dump 本身的 1.3~1.8 秒 —— 界面只要在动
(信息流/视频/加载动画),框就会落在旧位置上("抓取错位")。
同一个连接背靠背取把这个窗口压到最小,并额外用**双截图校验**标出
"抓取期间界面在变化"(`unstable`),前端据此明确提示而不是悄悄给错位的框。
?serial=xxx
→ {ok, image(data-url), width, height, elements:[...], unstable, cost_ms}
"""
serial = request.args.get("serial", "")
if not serial:
return jsonify({"ok": False, "error": "缺少 serial"}), 400
ok, data = uiauto_helper.snapshot(serial)
if not ok:
# 设备没连上/抓取失败:502(依赖不可用),与 /api/uiauto/elements 一致
return jsonify({"ok": False, "error": data}), 502
return jsonify({"ok": True, **data})
@bp.route("/api/uiauto/screenshot")
@perm_required(PERM_DEVICES)
def api_uiauto_screenshot():
"""通过 uiauto2 获取设备截图(供抓取元素时显示设备画面)。
?serial=xxx 设备 serial
直接返回 image/jpeg,前端用 <img> 加载。
"""
serial = request.args.get("serial", "")
if not serial:
return jsonify({"ok": False, "error": "缺少 serial"}), 400
ok, data = uiauto_helper.get_screenshot(serial)
if ok:
return Response(data, mimetype="image/jpeg",
headers={"Cache-Control": "no-store"})
return jsonify({"ok": False, "error": data}), 503
# ================== API:远程看屏(阶段 2) ==================
# MJPEG 实时画面流 + u2 触控注入。数据源 u2(atx-agent minicap 截图,单帧 ~200ms),
# 零新依赖(原计划 ws-scrcpy 不在 npm 分发,GitHub 下载在国内不可靠,改自建)。
# 坐标约定:前端以 <img> 显示尺寸归一化后映射到设备原生分辨率(naturalWidth/Height)。
_SCREEN_JPEG_QUALITY = 65
_SCREEN_FRAME_GAP = 0.05 # 帧间最小间隔(秒),防止空转烧 CPU
def _job_next_run(job):
"""任务下次执行时间,格式化为 "YYYY-MM-DD HH:MM"(Flask 序列化 aware datetime 会变 GMT 格式)。"""
nr = context.mgr.next_run_of(job)
return nr.strftime("%Y-%m-%d %H:%M") if nr else None