// ================== 动作录制器 ==================
// 在设备截图上操作(点/划/按键/输入),自动翻译成**步骤**:
// · 点 → 命中元素就记「点击元素」(选择器),抓不到就退化成「点击坐标」(百分比)
// · 划 → 记**「录制手势」**:完整轨迹点列 [[x,y,t_ms],…],**纯回放**,不再套滑动逻辑
// · 停顿 → 可选记成「等待」(按真实间隔 ±15%)
//
// 两个录制来源:
// · 网页上录:在这个窗口里按住鼠标拖(采样路径 + 时间)
// · 手机上录:点「手机上录」后**用手指在真机上划**——后端读 getevent 抓真触屏,
// 录到的是人手的真实轨迹(比在网页上模拟更真)
//
// 两种用法(同一个窗口):
// mode='action' —— 录一段,存成「自定义动作」(进动作库,能拖进任何任务)
// mode='gesture' —— 录到的手势直接回填给调用方(步骤编辑器里的「录制手势」步骤)
//
// 元素树用 /api/uiauto/snapshot 一次取齐(截图 + 元素),**每次操作后自动刷新**,
// 这样下一次点击就能用最新的树翻译选择器(抖音这种没有 id 的界面靠 text/desc)。
let _rec = null;
function _recEl() { return document.getElementById('rec-overlay'); }
async function openRecorder(opts) {
opts = opts || {};
if (_recEl()) return;
_rec = {
mode: opts.mode || 'action',
serial: opts.serial || '',
onDone: opts.onDone || null,
steps: [],
elements: [],
devW: 0, devH: 0, imgURL: '',
lastTs: 0, dragFrom: null, busy: false, keepPause: true,
samples: [], phone: null, autoPhone: !!opts.autoStartPhone,
shots: 0, unstable: false,
};
const ov = document.createElement('div');
ov.id = 'rec-overlay';
ov.className = 'modal-overlay show'; // .modal-overlay 默认 display:none,必须带 show
ov.innerHTML =
'
' +
'
' +
'' + (_rec.mode === 'gesture' ? '录制手势' : '录制动作') + '' +
'' +
'' +
'' +
'' +
'' +
'
' +
'
' +
'
' +
'
' +
'
' +
'' +
'' +
'' +
'' +
'
' +
'
' +
'
' +
'
' +
'' +
'' +
'
' +
'
录到的步骤
' +
'
' +
'' +
'
' +
'
' +
'
';
ov.addEventListener('mousedown', function (e) { if (e.target === ov) closeRecorder(); });
document.body.appendChild(ov);
await recLoadDevices();
recRender();
if (_rec.autoPhone && _rec.serial) recStartPhone();
}
function closeRecorder() {
const ov = _recEl();
if (ov) ov.remove();
_rec = null;
}
async function recLoadDevices() {
const sel = document.getElementById('rec-device');
if (!sel) return;
sel.innerHTML = '';
const r = await apiGet('/api/devices');
if (!r || !r.ok) return;
(r.items || []).forEach(function (d) {
const o = document.createElement('option');
o.value = d.serial;
o.textContent = d.name ? (d.name + '(' + d.serial + ')') : d.serial;
sel.appendChild(o);
});
if (_rec.serial) { sel.value = _rec.serial; }
else if ((r.items || []).length) { sel.value = r.items[0].serial; _rec.serial = r.items[0].serial; }
if (_rec.serial) { await recWake(); recRefresh(); }
}
async function recPickDevice() {
const sel = document.getElementById('rec-device');
_rec.serial = sel ? sel.value : '';
_rec.elements = [];
recRender();
if (_rec.serial) { await recWake(); recRefresh(); }
}
// 录制前先唤醒设备:**息屏时 u2 抓 UI 树会退化到几十秒**(实测 65s,唤醒后 3.3s),
// 截图也是一片黑。唤醒是幂等的,多调一次没有副作用。
async function recWake() {
if (!_rec || !_rec.serial) return;
try {
const r = await apiPost('/api/device/screen_all', { mode: 'on', serials: [_rec.serial] });
if (r && r.ok) await new Promise(function (z) { setTimeout(z, 800); });
} catch (e) { /* 唤醒失败也继续,用户自己会看到黑屏 */ }
}
// 取一张快照:截图 + 元素树(同一个连接背靠背取,避免框错位)
async function recRefresh() {
if (!_rec || !_rec.serial) return;
const r = await apiGet('/api/uiauto/snapshot?serial=' + encodeURIComponent(_rec.serial));
if (!r || !r.ok) {
const st = document.getElementById('rec-stage');
if (st) st.innerHTML = '取画面失败:' + esc((r && r.error) || '未知错误') + '
';
return;
}
_rec.imgURL = r.image || '';
_rec.devW = r.width || 0;
_rec.devH = r.height || 0;
_rec.elements = r.elements || [];
_rec.unstable = !!r.unstable;
_rec.shots++;
recPaint();
}
function recPaint() {
const st = document.getElementById('rec-stage');
if (!st) return;
if (!_rec.imgURL) { st.innerHTML = '正在取画面…
'; return; }
st.innerHTML = '
' +
(_rec.unstable ? '画面在变化,这次框可能不准
' : '');
const img = document.getElementById('rec-img');
img.addEventListener('mousedown', recDown);
img.addEventListener('mousemove', recMove);
img.addEventListener('mouseup', recUp);
img.addEventListener('mouseleave', function () { _rec.dragFrom = null; recPaintOverlay(); });
recPaintOverlay();
}
// 拖动中的轨迹提示(录制时能看见自己划到哪)
function recPaintOverlay() {
const st = document.getElementById('rec-stage');
if (!st) return;
let cv = document.getElementById('rec-overlay-line');
if (!cv) {
cv = document.createElement('div');
cv.id = 'rec-overlay-line';
cv.className = 'rec-line';
st.appendChild(cv);
}
if (!_rec.dragFrom) { cv.style.display = 'none'; return; }
cv.style.display = 'block';
const a = _rec.dragFrom;
const b = _rec.dragTo || a;
const x = Math.min(a.x, b.x), y = Math.min(a.y, b.y);
cv.style.left = x + 'px'; cv.style.top = y + 'px';
cv.style.width = Math.abs(b.x - a.x) + 'px';
cv.style.height = Math.abs(b.y - a.y) + 'px';
}
function _recPos(e) {
const img = document.getElementById('rec-img');
if (!img) return null;
const r = img.getBoundingClientRect();
const x = e.clientX - r.left, y = e.clientY - r.top;
if (x < 0 || y < 0 || x > r.width || y > r.height) return null;
// 显示坐标 → 设备坐标
const sx = _rec.devW / r.width, sy = _rec.devH / r.height;
return { x: Math.round(x * sx), y: Math.round(y * sy), rx: x, ry: y, cssW: r.width, cssH: r.height };
}
function recDown(e) {
if (_rec.busy) return;
const p = _recPos(e);
if (!p) return;
_rec.dragFrom = p; _rec.dragTo = null; _rec.dragT0 = Date.now();
// 采样:拖动过程中把每个点连时间一起记下来(这就是"轨迹")
_rec.samples = [[p.x, p.y, 0]];
recPaintOverlay();
}
function recMove(e) {
if (!_rec.dragFrom) return;
const p = _recPos(e);
if (!p) return;
_rec.dragTo = p;
const t = Date.now() - _rec.dragT0;
const last = _rec.samples[_rec.samples.length - 1];
if (!last || t - last[2] >= 8 || Math.abs(p.x - last[0]) + Math.abs(p.y - last[1]) >= 3) {
if (_rec.samples.length < 400) _rec.samples.push([p.x, p.y, t]);
}
recPaintOverlay();
}
async function recUp(e) {
if (!_rec.dragFrom || _rec.busy) { _rec.dragFrom = null; return; }
const a = _rec.dragFrom;
const b = _recPos(e) || a;
const durMs = Date.now() - _rec.dragT0;
_rec.dragFrom = null;
recPaintOverlay();
const dist = Math.hypot(b.x - a.x, b.y - a.y);
if (dist < 12 || durMs < 60) { // 视为点击
_rec.samples = [];
await recTap(a.x, a.y);
} else {
const pts = _rec.samples.slice();
pts.push([b.x, b.y, durMs]); // 末点补上(决定手势收尾位置)
_rec.samples = [];
recPushGesture(pts, 'web');
await recRefresh();
}
}
// 一条录到的轨迹 → 「录制手势」步骤(**纯回放**:点列+时长原样存下,不再套滑动逻辑)
function recPushGesture(points, src) {
if (!points || points.length < 2) { showToast('没录到有效轨迹(太短了)', 'error'); return; }
if (_rec.mode === 'gesture') { // 回填模式:直接交给调用方,收工
const cb = _rec.onDone;
closeRecorder();
if (cb) cb({ points: points, speed: 1.0 }, src);
return;
}
const dur = points[points.length - 1][2] || 0;
_recPush({ type: 'gesture',
label: '录制手势 ' + points.length + '点/' + (dur / 1000).toFixed(1) + 's',
params: { points: points, speed: 1.0 } });
}
// ---------- 手机端录制(getevent 抓真手指) ----------
async function recStartPhone() {
if (!_rec.serial) { showToast('先选设备', 'error'); return; }
const r = await apiPost('/api/gesture/record/start', { serial: _rec.serial });
if (!r || !r.ok) { showToast((r && r.error) || '开始失败', 'error'); return; }
_rec.phone = { token: r.token, t0: Date.now() };
showToast('开始录制——现在用手指在手机上划', 'success');
recRenderFoot();
}
async function recStopPhone() {
if (!_rec.phone) return;
const token = _rec.phone.token;
_rec.phone = null;
const r = await apiPost('/api/gesture/record/stop', { token: token });
if (!r || !r.ok) { showToast((r && r.error) || '停止失败', 'error'); recRenderFoot(); return; }
const gs = r.gestures || [];
if (!gs.length) {
showToast('没录到手势——请在手机屏幕上用指划一下(不是点)', 'error');
recRenderFoot(); return;
}
if (_rec.mode === 'gesture') { recPushGesture(gs[gs.length - 1], 'phone'); return; }
gs.forEach(function (g) { recPushGesture(g, 'phone'); });
showToast('录到 ' + gs.length + ' 个手势', 'success');
recRenderFoot();
recRefresh();
}
// ---------- 翻译:点击 → 步骤 ----------
function _recFindElement(x, y) {
// 取包含该点的最小元素;优先可点击的(和 /api/screen/tap 的 snap 同一口径)
let best = null, bestClick = null;
(_rec.elements || []).forEach(function (el) {
const m = (el.bounds || '').match(/\[(\d+),(\d+)\]\[(\d+),(\d+)\]/);
if (!m) return;
const x1 = +m[1], y1 = +m[2], x2 = +m[3], y2 = +m[4];
if (!(x1 <= x && x <= x2 && y1 <= y && y <= y2)) return;
const area = (x2 - x1) * (y2 - y1);
const cand = { el: el, area: area, cx: Math.round((x1 + x2) / 2), cy: Math.round((y1 + y2) / 2) };
if (!best || area < best.area) best = cand;
if (String(el.clickable) === 'true' && (!bestClick || area < bestClick.area)) bestClick = cand;
});
return bestClick || best;
}
async function recTap(x, y) {
if (!_rec || _rec.busy) return;
_rec.busy = true;
try {
const hit = _recFindElement(x, y);
let step;
if (hit && hit.el.suggested && hit.el.suggested.value) {
step = { type: 'click', label: '点击' + _recShort(hit.el),
params: { selector_type: hit.el.suggested.type || 'xpath',
selector_value: hit.el.suggested.value, wait_timeout: 2 } };
} else {
step = { type: 'click_xy', label: '点击坐标',
params: { x: Math.round(x / _rec.devW * 100), y: Math.round(y / _rec.devH * 100) } };
}
_recPush(step);
const r = await apiPost('/api/screen/tap', { serial: _rec.serial, x: x, y: y, snap: 1 });
if (r && !r.ok) showToast((r.error || '点击失败'), 'error');
await recRefresh();
} finally { _rec.busy = false; }
}
function _recShort(el) {
const s = el.text || el.description || el.name || el.resource_id || '';
return s ? ('「' + String(s).slice(0, 12) + '」') : '';
}
async function recKey(key) {
if (!_rec || !_rec.serial) { showToast('先选设备', 'error'); return; }
_recPush({ type: 'key_event', label: '按键 ' + key, params: { key: key } });
const r = await apiPost('/api/screen/key', { serial: _rec.serial, key: key });
if (r && !r.ok) showToast((r.error || '按键失败'), 'error');
await recRefresh();
}
async function recText() {
if (!_rec || !_rec.serial) { showToast('先选设备', 'error'); return; }
const inp = document.getElementById('rec-text');
const t = (inp ? inp.value : '').trim();
if (!t) { showToast('先填要输入的文字', 'error'); return; }
_recPush({ type: 'input_text', label: '输入 ' + t.slice(0, 8),
params: { mode: 'fixed', fixed_text: t, clear_first: false } });
const r = await apiPost('/api/screen/text', { serial: _rec.serial, text: t });
if (r && !r.ok) showToast((r.error || '输入失败'), 'error');
if (inp) inp.value = '';
await recRefresh();
}
function recAddWait() {
_recPush({ type: 'wait', label: '等待', params: { min: 1, max: 3 } });
}
// 入列:按需自动补一条「等待」(真实停顿,±15%),并刷新列表
function _recPush(step) {
const now = Date.now();
if (_rec.keepPause && _rec.lastTs && step.type !== 'wait') {
const gap = (now - _rec.lastTs) / 1000;
if (gap >= 1.0) {
_rec.steps.push({ id: 'rec_' + Math.random().toString(36).slice(2, 8),
type: 'wait', label: '等待 ' + gap.toFixed(1) + 's',
params: { min: +(gap * 0.85).toFixed(1), max: +(gap * 1.15).toFixed(1) } });
}
}
step.id = 'rec_' + Math.random().toString(36).slice(2, 8);
_rec.steps.push(step);
_rec.lastTs = now;
recRender();
}
function recClear() { _rec.steps = []; _rec.lastTs = 0; recRender(); }
function recDel(i) { _rec.steps.splice(i, 1); recRender(); }
function recMove2(i, d) {
const j = i + d;
if (j < 0 || j >= _rec.steps.length) return;
const t = _rec.steps[i]; _rec.steps[i] = _rec.steps[j]; _rec.steps[j] = t;
recRender();
}
function recRender() {
const list = document.getElementById('rec-list');
if (!list) return;
const steps = _rec ? _rec.steps : [];
document.getElementById('rec-count').textContent = steps.length ? ('共 ' + steps.length + ' 步') : '';
list.innerHTML = steps.map(function (s, i) {
return '' + (i + 1) + '' +
'' + esc(s.label || s.type) + '' +
'' +
'' +
'' +
'' +
'
';
}).join('') || '还没有录到步骤——在左边画面上点一下或划一下试试
';
recRenderFoot();
}
function recRenderFoot() {
const foot = document.getElementById('rec-foot');
if (!foot) return;
const steps = _rec.steps;
if (_rec.mode === 'gesture') {
// 回填模式:录到就立刻填回那个步骤,不用再点确认
foot.innerHTML = _rec.phone
? ''
: '' +
'或在左边画面上按住鼠标拖一下——录到就立刻填进那一步
';
return;
}
const phoneBtn = _rec.phone
? ''
: '';
foot.innerHTML =
'' + phoneBtn +
'' + (_rec.phone
? '正在录:请用手指在手机屏幕上划,划完点「停止」'
: '手机上用手指划 = 录真手指轨迹;在左边画面上拖 = 网页录制') + '
' +
'' +
'
' +
'划出来的每一段都是一条「录制手势」步骤(按原路径与时长回放,不再套滑动逻辑)
';
}
async function recSaveAsAction() {
const el = document.getElementById('rec-name');
const name = (el ? el.value : '').trim();
if (!name) { showToast('先给动作起个名', 'error'); return; }
const steps = _rec.steps.filter(function (s) { return s.type !== 'wait' || true; })
.map(function (s) { return { type: s.type, label: s.label, params: s.params }; });
if (!steps.length) { showToast('还没有录到步骤', 'error'); return; }
const r = await apiPost('/api/custom_actions', { name: name, icon: '⏺', steps: steps });
if (!r || !r.ok) { showToast((r && r.error) || '保存失败', 'error'); return; }
showToast('已存成动作「' + name + '」(' + steps.length + ' 步)', 'success');
if (typeof loadCustomActions === 'function') loadCustomActions();
closeRecorder();
}