butubb
bd51306a2a
需求：滑动要能"我自己录制"；在自定义动作旁边加一个分栏，能配动作默认值、
能录制操作当动作、能自己建立动作。
**步骤默认值**（core/step_defaults.py，存 app_meta.step_defaults 单键，不建表）：
- 覆盖 滑动（方向/时长区间/幅度/抖动/拟人）、点击（等待超时）、长按（时长/超时）、
等待（区间）、按键、输入文字（模式/候选文案/清空）；
- **只影响之后新建的步骤**，已有步骤不动；新建时 `_makeStep` 用出厂值打底 + 默认值覆盖；
- 只存与出厂值不同的字段（以后调出厂默认，没配过的能跟着走）；
- 字段白名单 + 范围夹取（SPEC）：越界夹边界、非法值丢弃 → 手改 JSON 塞脏值也进不来。
**动作录制器**（static/admin/recorder.js）：
- 在设备画面上点/划/按键/输入 → 自动翻译成步骤：
点→命中元素记「点击元素」(选择器)、没命中退化成「点击坐标」(百分比)；
划→「滑动」方向/幅度/时长全按你实际操作算；停顿≥1s 可选记成「等待」(±15%)；
- 两个入口：动作配置页「录制动作」（存成自定义动作，进动作库复用）、
步骤编辑器滑动那步的「录制手势」（只取手势回填该步骤）；
- 画面+元素树走 /api/uiauto/snapshot 一次取齐，**每次操作后自动刷新**，
所以下一次点击用的是最新的树；执行仍走 /api/screen/{tap,swipe,key,text}（与大屏同一套）。
接口：GET/POST /api/step_defaults（读登录、写需 tasks 权限）。
文档：TASK_DEV §3.1（含翻译规则表与三条实现口径）、API §2.3、DATA_MODEL §5、README。
自测（无头浏览器 + CDP 真模拟拖动）：默认值保存→回读→新建滑动步骤确实预填 0.42；
录制器取到画面与元素树；模拟"按住往上划"→ 录出
{direction:up, distance_ratio:0.5, duration_min:0.47, humanize:true}（幅度/时长按实际操作算）；
恢复出厂链路 0.6 通过；页面无 JS 报错。
376 lines
16 KiB
JavaScript
376 lines
16 KiB
JavaScript
// ================== 动作录制器 ==================
|
||
// 在设备截图上操作（点/划/按键/输入），自动翻译成**步骤**：
|
||
// · 点 → 命中元素就记「点击元素」(选择器)，抓不到就退化成「点击坐标」(百分比)
|
||
// · 划 → 记「滑动」：方向 + 幅度(屏占比) + 实际时长
|
||
// · 停顿 → 可选记成「等待」（按真实间隔 ±15%）
|
||
//
|
||
// 两种用法（同一个录窗口）：
|
||
// mode='action' —— 录一段，存成「自定义动作」（进动作库，能拖进任何任务）
|
||
// mode='gesture' —— 只取最后一个手势，回填当前滑动步骤的参数
|
||
//
|
||
// 元素树用 /api/uiauto/snapshot 一次取齐（截图 + 元素），**每次操作后自动刷新**，
|
||
// 这样下一次点击就能用最新的树翻译选择器（抖音这种没有 id 的界面靠 text/desc）。
|
||
|
||
let _rec = null;
|
||
|
||
function _recEl() { return document.getElementById('rec-overlay'); }
|
||
|
||
async function openRecorder(opts) {
|
||
opts = opts || {};
|
||
if (_recEl()) return;
|
||
_rec = {
|
||
mode: opts.mode || 'action',
|
||
serial: opts.serial || '',
|
||
onDone: opts.onDone || null,
|
||
steps: [],
|
||
elements: [],
|
||
devW: 0, devH: 0, imgURL: '',
|
||
lastTs: 0, dragFrom: null, busy: false, keepPause: true,
|
||
shots: 0, unstable: false,
|
||
};
|
||
const ov = document.createElement('div');
|
||
ov.id = 'rec-overlay';
|
||
ov.className = 'modal-overlay show'; // .modal-overlay 默认 display:none，必须带 show
|
||
ov.innerHTML =
|
||
'<div class="modal-box rec-box">' +
|
||
'<div class="rec-head">' +
|
||
'<b>' + (_rec.mode === 'gesture' ? '录制手势' : '录制动作') + '</b>' +
|
||
'<select id="rec-device" class="form-control" style="width:auto" onchange="recPickDevice()"></select>' +
|
||
'<button class="btn btn-sm" onclick="recRefresh()">刷新画面</button>' +
|
||
'<label class="form-check"><input type="checkbox" id="rec-pause" checked onchange="_rec.keepPause=this.checked"><span>记录停顿</span></label>' +
|
||
'<span style="flex:1"></span>' +
|
||
'<button class="btn btn-sm" onclick="closeRecorder()">关闭</button>' +
|
||
'</div>' +
|
||
'<div class="rec-body">' +
|
||
'<div class="rec-stage" id="rec-stage"><div class="rec-hint">正在取画面…</div></div>' +
|
||
'<div class="rec-side">' +
|
||
'<div class="rec-ops">' +
|
||
'<button class="btn btn-sm" onclick="recKey(\'back\')">返回</button>' +
|
||
'<button class="btn btn-sm" onclick="recKey(\'home\')">Home</button>' +
|
||
'<button class="btn btn-sm" onclick="recKey(\'recent\')">最近</button>' +
|
||
'<button class="btn btn-sm" onclick="recKey(\'enter\')">回车</button>' +
|
||
'</div>' +
|
||
'<div class="rec-row"><input id="rec-text" class="form-control" placeholder="要输入的文字">' +
|
||
'<button class="btn btn-sm" onclick="recText()">输入</button></div>' +
|
||
'<div class="rec-ops">' +
|
||
'<button class="btn btn-sm" onclick="recAddWait()">+ 等待</button>' +
|
||
'<button class="btn btn-sm" onclick="recClear()">清空</button>' +
|
||
'</div>' +
|
||
'<div class="rec-list-title">录到的步骤 <span id="rec-count" class="text-muted"></span></div>' +
|
||
'<div class="rec-list" id="rec-list"></div>' +
|
||
'<div class="rec-foot" id="rec-foot"></div>' +
|
||
'</div>' +
|
||
'</div>' +
|
||
'</div>';
|
||
ov.addEventListener('mousedown', function (e) { if (e.target === ov) closeRecorder(); });
|
||
document.body.appendChild(ov);
|
||
await recLoadDevices();
|
||
recRender();
|
||
}
|
||
|
||
function closeRecorder() {
|
||
const ov = _recEl();
|
||
if (ov) ov.remove();
|
||
_rec = null;
|
||
}
|
||
|
||
async function recLoadDevices() {
|
||
const sel = document.getElementById('rec-device');
|
||
if (!sel) return;
|
||
sel.innerHTML = '<option value="">选择设备…</option>';
|
||
const r = await apiGet('/api/devices');
|
||
if (!r || !r.ok) return;
|
||
(r.items || []).forEach(function (d) {
|
||
const o = document.createElement('option');
|
||
o.value = d.serial;
|
||
o.textContent = d.name ? (d.name + '（' + d.serial + '）') : d.serial;
|
||
sel.appendChild(o);
|
||
});
|
||
if (_rec.serial) { sel.value = _rec.serial; }
|
||
else if ((r.items || []).length) { sel.value = r.items[0].serial; _rec.serial = r.items[0].serial; }
|
||
if (_rec.serial) recRefresh();
|
||
}
|
||
|
||
function recPickDevice() {
|
||
const sel = document.getElementById('rec-device');
|
||
_rec.serial = sel ? sel.value : '';
|
||
_rec.elements = [];
|
||
recRender();
|
||
if (_rec.serial) recRefresh();
|
||
}
|
||
|
||
// 取一张快照：截图 + 元素树（同一个连接背靠背取，避免框错位）
|
||
async function recRefresh() {
|
||
if (!_rec || !_rec.serial) return;
|
||
const r = await apiGet('/api/uiauto/snapshot?serial=' + encodeURIComponent(_rec.serial));
|
||
if (!r || !r.ok) {
|
||
const st = document.getElementById('rec-stage');
|
||
if (st) st.innerHTML = '<div class="rec-hint">取画面失败：' + esc((r && r.error) || '未知错误') + '</div>';
|
||
return;
|
||
}
|
||
_rec.imgURL = r.image || '';
|
||
_rec.devW = r.width || 0;
|
||
_rec.devH = r.height || 0;
|
||
_rec.elements = r.elements || [];
|
||
_rec.unstable = !!r.unstable;
|
||
_rec.shots++;
|
||
recPaint();
|
||
}
|
||
|
||
function recPaint() {
|
||
const st = document.getElementById('rec-stage');
|
||
if (!st) return;
|
||
if (!_rec.imgURL) { st.innerHTML = '<div class="rec-hint">正在取画面…</div>'; return; }
|
||
st.innerHTML = '<img id="rec-img" src="' + _rec.imgURL + '" draggable="false">' +
|
||
(_rec.unstable ? '<div class="rec-warn">画面在变化，这次框可能不准</div>' : '');
|
||
const img = document.getElementById('rec-img');
|
||
img.addEventListener('mousedown', recDown);
|
||
img.addEventListener('mousemove', recMove);
|
||
img.addEventListener('mouseup', recUp);
|
||
img.addEventListener('mouseleave', function () { _rec.dragFrom = null; recPaintOverlay(); });
|
||
recPaintOverlay();
|
||
}
|
||
|
||
// 拖动中的轨迹提示（录制时能看见自己划到哪）
|
||
function recPaintOverlay() {
|
||
const st = document.getElementById('rec-stage');
|
||
if (!st) return;
|
||
let cv = document.getElementById('rec-overlay-line');
|
||
if (!cv) {
|
||
cv = document.createElement('div');
|
||
cv.id = 'rec-overlay-line';
|
||
cv.className = 'rec-line';
|
||
st.appendChild(cv);
|
||
}
|
||
if (!_rec.dragFrom) { cv.style.display = 'none'; return; }
|
||
cv.style.display = 'block';
|
||
const a = _rec.dragFrom;
|
||
const b = _rec.dragTo || a;
|
||
const x = Math.min(a.x, b.x), y = Math.min(a.y, b.y);
|
||
cv.style.left = x + 'px'; cv.style.top = y + 'px';
|
||
cv.style.width = Math.abs(b.x - a.x) + 'px';
|
||
cv.style.height = Math.abs(b.y - a.y) + 'px';
|
||
}
|
||
|
||
function _recPos(e) {
|
||
const img = document.getElementById('rec-img');
|
||
if (!img) return null;
|
||
const r = img.getBoundingClientRect();
|
||
const x = e.clientX - r.left, y = e.clientY - r.top;
|
||
if (x < 0 || y < 0 || x > r.width || y > r.height) return null;
|
||
// 显示坐标 → 设备坐标
|
||
const sx = _rec.devW / r.width, sy = _rec.devH / r.height;
|
||
return { x: Math.round(x * sx), y: Math.round(y * sy), rx: x, ry: y, cssW: r.width, cssH: r.height };
|
||
}
|
||
|
||
function recDown(e) {
|
||
if (_rec.busy) return;
|
||
const p = _recPos(e);
|
||
if (!p) return;
|
||
_rec.dragFrom = p; _rec.dragTo = null; _rec.dragT0 = Date.now();
|
||
recPaintOverlay();
|
||
}
|
||
|
||
function recMove(e) {
|
||
if (!_rec.dragFrom) return;
|
||
const p = _recPos(e);
|
||
if (!p) return;
|
||
_rec.dragTo = p;
|
||
recPaintOverlay();
|
||
}
|
||
|
||
async function recUp(e) {
|
||
if (!_rec.dragFrom || _rec.busy) { _rec.dragFrom = null; return; }
|
||
const a = _rec.dragFrom;
|
||
const b = _recPos(e) || a;
|
||
const durMs = Date.now() - _rec.dragT0;
|
||
_rec.dragFrom = null;
|
||
recPaintOverlay();
|
||
const dist = Math.hypot(b.x - a.x, b.y - a.y);
|
||
if (dist < 12 || durMs < 60) { // 视为点击
|
||
await recTap(a.x, a.y);
|
||
} else {
|
||
await recSwipe(a, b, durMs);
|
||
}
|
||
}
|
||
|
||
// ---------- 翻译：点击 → 步骤 ----------
|
||
function _recFindElement(x, y) {
|
||
// 取包含该点的最小元素；优先可点击的（和 /api/screen/tap 的 snap 同一口径）
|
||
let best = null, bestClick = null;
|
||
(_rec.elements || []).forEach(function (el) {
|
||
const m = (el.bounds || '').match(/\[(\d+),(\d+)\]\[(\d+),(\d+)\]/);
|
||
if (!m) return;
|
||
const x1 = +m[1], y1 = +m[2], x2 = +m[3], y2 = +m[4];
|
||
if (!(x1 <= x && x <= x2 && y1 <= y && y <= y2)) return;
|
||
const area = (x2 - x1) * (y2 - y1);
|
||
const cand = { el: el, area: area, cx: Math.round((x1 + x2) / 2), cy: Math.round((y1 + y2) / 2) };
|
||
if (!best || area < best.area) best = cand;
|
||
if (String(el.clickable) === 'true' && (!bestClick || area < bestClick.area)) bestClick = cand;
|
||
});
|
||
return bestClick || best;
|
||
}
|
||
|
||
async function recTap(x, y) {
|
||
if (!_rec || _rec.busy) return;
|
||
_rec.busy = true;
|
||
try {
|
||
const hit = _recFindElement(x, y);
|
||
let step;
|
||
if (hit && hit.el.suggested && hit.el.suggested.value) {
|
||
step = { type: 'click', label: '点击' + _recShort(hit.el),
|
||
params: { selector_type: hit.el.suggested.type || 'xpath',
|
||
selector_value: hit.el.suggested.value, wait_timeout: 2 } };
|
||
} else {
|
||
step = { type: 'click_xy', label: '点击坐标',
|
||
params: { x: Math.round(x / _rec.devW * 100), y: Math.round(y / _rec.devH * 100) } };
|
||
}
|
||
_recPush(step);
|
||
const r = await apiPost('/api/screen/tap', { serial: _rec.serial, x: x, y: y, snap: 1 });
|
||
if (r && !r.ok) showToast((r.error || '点击失败'), 'error');
|
||
await recRefresh();
|
||
} finally { _rec.busy = false; }
|
||
}
|
||
|
||
function _recShort(el) {
|
||
const s = el.text || el.description || el.name || el.resource_id || '';
|
||
return s ? ('「' + String(s).slice(0, 12) + '」') : '';
|
||
}
|
||
|
||
// ---------- 翻译：滑动 → 步骤 ----------
|
||
async function recSwipe(a, b, durMs) {
|
||
if (!_rec || _rec.busy) return;
|
||
_rec.busy = true;
|
||
try {
|
||
const dx = b.x - a.x, dy = b.y - a.y;
|
||
const vertical = Math.abs(dy) >= Math.abs(dx);
|
||
const direction = vertical ? (dy < 0 ? 'up' : 'down') : (dx < 0 ? 'left' : 'right');
|
||
const span = Math.abs(vertical ? dy : dx);
|
||
const base = vertical ? _rec.devH : _rec.devW;
|
||
const ratio = Math.max(0.05, Math.min(0.9, span / base));
|
||
const dur = Math.max(0.08, durMs / 1000);
|
||
const step = { type: 'swipe', label: '滑动 ' + direction,
|
||
params: { direction: direction,
|
||
duration_min: +(dur * 0.9).toFixed(2),
|
||
duration_max: +(dur * 1.15).toFixed(2),
|
||
distance_ratio: +ratio.toFixed(2),
|
||
jitter: 0.15, humanize: true } };
|
||
_recPush(step);
|
||
const r = await apiPost('/api/screen/swipe',
|
||
{ serial: _rec.serial, x1: a.x, y1: a.y, x2: b.x, y2: b.y,
|
||
duration: Math.round(durMs) / 1000 });
|
||
if (r && !r.ok) showToast((r.error || '滑动失败'), 'error');
|
||
await recRefresh();
|
||
} finally { _rec.busy = false; }
|
||
}
|
||
|
||
async function recKey(key) {
|
||
if (!_rec || !_rec.serial) { showToast('先选设备', 'error'); return; }
|
||
_recPush({ type: 'key_event', label: '按键 ' + key, params: { key: key } });
|
||
const r = await apiPost('/api/screen/key', { serial: _rec.serial, key: key });
|
||
if (r && !r.ok) showToast((r.error || '按键失败'), 'error');
|
||
await recRefresh();
|
||
}
|
||
|
||
async function recText() {
|
||
if (!_rec || !_rec.serial) { showToast('先选设备', 'error'); return; }
|
||
const inp = document.getElementById('rec-text');
|
||
const t = (inp ? inp.value : '').trim();
|
||
if (!t) { showToast('先填要输入的文字', 'error'); return; }
|
||
_recPush({ type: 'input_text', label: '输入 ' + t.slice(0, 8),
|
||
params: { mode: 'fixed', fixed_text: t, clear_first: false } });
|
||
const r = await apiPost('/api/screen/text', { serial: _rec.serial, text: t });
|
||
if (r && !r.ok) showToast((r.error || '输入失败'), 'error');
|
||
if (inp) inp.value = '';
|
||
await recRefresh();
|
||
}
|
||
|
||
function recAddWait() {
|
||
_recPush({ type: 'wait', label: '等待', params: { min: 1, max: 3 } });
|
||
}
|
||
|
||
// 入列：按需自动补一条「等待」（真实停顿，±15%），并刷新列表
|
||
function _recPush(step) {
|
||
const now = Date.now();
|
||
if (_rec.keepPause && _rec.lastTs && step.type !== 'wait') {
|
||
const gap = (now - _rec.lastTs) / 1000;
|
||
if (gap >= 1.0) {
|
||
_rec.steps.push({ id: 'rec_' + Math.random().toString(36).slice(2, 8),
|
||
type: 'wait', label: '等待 ' + gap.toFixed(1) + 's',
|
||
params: { min: +(gap * 0.85).toFixed(1), max: +(gap * 1.15).toFixed(1) } });
|
||
}
|
||
}
|
||
step.id = 'rec_' + Math.random().toString(36).slice(2, 8);
|
||
_rec.steps.push(step);
|
||
_rec.lastTs = now;
|
||
recRender();
|
||
}
|
||
|
||
function recClear() { _rec.steps = []; _rec.lastTs = 0; recRender(); }
|
||
|
||
function recDel(i) { _rec.steps.splice(i, 1); recRender(); }
|
||
function recMove2(i, d) {
|
||
const j = i + d;
|
||
if (j < 0 || j >= _rec.steps.length) return;
|
||
const t = _rec.steps[i]; _rec.steps[i] = _rec.steps[j]; _rec.steps[j] = t;
|
||
recRender();
|
||
}
|
||
|
||
function recRender() {
|
||
const list = document.getElementById('rec-list');
|
||
if (!list) return;
|
||
const steps = _rec ? _rec.steps : [];
|
||
document.getElementById('rec-count').textContent = steps.length ? ('共 ' + steps.length + ' 步') : '';
|
||
list.innerHTML = steps.map(function (s, i) {
|
||
return '<div class="rec-item"><span class="rec-idx">' + (i + 1) + '</span>' +
|
||
'<span class="rec-label">' + esc(s.label || s.type) + '</span>' +
|
||
'<span class="rec-ops2">' +
|
||
'<button class="btn btn-xs" onclick="recMove2(' + i + ',-1)" title="上移">↑</button>' +
|
||
'<button class="btn btn-xs" onclick="recMove2(' + i + ',1)" title="下移">↓</button>' +
|
||
'<button class="btn btn-xs btn-danger" onclick="recDel(' + i + ')">×</button>' +
|
||
'</span></div>';
|
||
}).join('') || '<div class="text-muted" style="font-size:12px;padding:6px 0">还没有录到步骤——在左边画面上点一下或划一下试试</div>';
|
||
recRenderFoot();
|
||
}
|
||
|
||
function recRenderFoot() {
|
||
const foot = document.getElementById('rec-foot');
|
||
if (!foot) return;
|
||
const steps = _rec.steps;
|
||
if (_rec.mode === 'gesture') {
|
||
const sw = steps.filter(function (s) { return s.type === 'swipe'; });
|
||
foot.innerHTML = sw.length
|
||
? '<button class="btn btn-primary" onclick="recApplyGesture()">用这个手势（共录到 ' + sw.length + ' 个滑动，取最后一个）</button>' +
|
||
'<div class="help">把方向/幅度/时长回填到当前滑动步骤</div>'
|
||
: '<div class="text-muted" style="font-size:12px">在画面上划一下（按住拖动），然后点这里</div>';
|
||
return;
|
||
}
|
||
foot.innerHTML =
|
||
'<div class="rec-row"><input id="rec-name" class="form-control" placeholder="动作名，如「抖音-进评论」">' +
|
||
'<button class="btn btn-primary" onclick="recSaveAsAction()">存成动作</button></div>' +
|
||
'<div class="help">存进「自定义动作」库，之后能拖进任何任务的步骤画布</div>';
|
||
}
|
||
|
||
function recApplyGesture() {
|
||
const sw = _rec.steps.filter(function (s) { return s.type === 'swipe'; });
|
||
if (!sw.length) return;
|
||
const p = sw[sw.length - 1].params;
|
||
const cb = _rec.onDone;
|
||
closeRecorder();
|
||
if (cb) cb(p);
|
||
}
|
||
|
||
async function recSaveAsAction() {
|
||
const el = document.getElementById('rec-name');
|
||
const name = (el ? el.value : '').trim();
|
||
if (!name) { showToast('先给动作起个名', 'error'); return; }
|
||
const steps = _rec.steps.filter(function (s) { return s.type !== 'wait' || true; })
|
||
.map(function (s) { return { type: s.type, label: s.label, params: s.params }; });
|
||
if (!steps.length) { showToast('还没有录到步骤', 'error'); return; }
|
||
const r = await apiPost('/api/custom_actions', { name: name, icon: '⏺', steps: steps });
|
||
if (!r || !r.ok) { showToast((r && r.error) || '保存失败', 'error'); return; }
|
||
showToast('已存成动作「' + name + '」（' + steps.length + ' 步）', 'success');
|
||
if (typeof loadCustomActions === 'function') loadCustomActions();
|
||
closeRecorder();
|
||
}
|