// ================== 动作录制器 ================== // 在设备截图上操作(点/划/按键/输入),自动翻译成**步骤**: // · 点 → 命中元素就记「点击元素」(选择器),抓不到就退化成「点击坐标」(百分比) // · 划 → 记**「录制手势」**:完整轨迹点列 [[x,y,t_ms],…],**纯回放**,不再套滑动逻辑 // · 停顿 → 可选记成「等待」(按真实间隔 ±15%) // // 两个录制来源: // · 网页上录:在这个窗口里按住鼠标拖(采样路径 + 时间) // · 手机上录:点「手机上录」后**用手指在真机上划**——后端读 getevent 抓真触屏, // 录到的是人手的真实轨迹(比在网页上模拟更真) // // 两种用法(同一个窗口): // mode='action' —— 录一段,存成「自定义动作」(进动作库,能拖进任何任务) // mode='gesture' —— 录到的手势直接回填给调用方(步骤编辑器里的「录制手势」步骤) // // 元素树用 /api/uiauto/snapshot 一次取齐(截图 + 元素),**每次操作后自动刷新**, // 这样下一次点击就能用最新的树翻译选择器(抖音这种没有 id 的界面靠 text/desc)。 let _rec = null; function _recEl() { return document.getElementById('rec-overlay'); } async function openRecorder(opts) { opts = opts || {}; if (_recEl()) return; _rec = { mode: opts.mode || 'action', serial: opts.serial || '', onDone: opts.onDone || null, steps: [], elements: [], devW: 0, devH: 0, imgURL: '', lastTs: 0, dragFrom: null, busy: false, keepPause: true, samples: [], phone: null, autoPhone: !!opts.autoStartPhone, shots: 0, unstable: false, }; const ov = document.createElement('div'); ov.id = 'rec-overlay'; ov.className = 'modal-overlay show'; // .modal-overlay 默认 display:none,必须带 show ov.innerHTML = ''; ov.addEventListener('mousedown', function (e) { if (e.target === ov) closeRecorder(); }); document.body.appendChild(ov); await recLoadDevices(); recRender(); if (_rec.autoPhone && _rec.serial) recStartPhone(); } function closeRecorder() { const ov = _recEl(); if (ov) ov.remove(); _rec = null; } async function recLoadDevices() { const sel = document.getElementById('rec-device'); if (!sel) return; sel.innerHTML = ''; const r = await apiGet('/api/devices'); if (!r || !r.ok) return; (r.items || []).forEach(function (d) { const o = document.createElement('option'); o.value = d.serial; o.textContent = d.name ? (d.name + '(' + d.serial + ')') : d.serial; sel.appendChild(o); }); if (_rec.serial) { sel.value = _rec.serial; } else if ((r.items || []).length) { sel.value = r.items[0].serial; _rec.serial = r.items[0].serial; } if (_rec.serial) { await recWake(); recRefresh(); } } async function recPickDevice() { const sel = document.getElementById('rec-device'); _rec.serial = sel ? sel.value : ''; _rec.elements = []; recRender(); if (_rec.serial) { await recWake(); recRefresh(); } } // 录制前先唤醒设备:**息屏时 u2 抓 UI 树会退化到几十秒**(实测 65s,唤醒后 3.3s), // 截图也是一片黑。唤醒是幂等的,多调一次没有副作用。 async function recWake() { if (!_rec || !_rec.serial) return; try { const r = await apiPost('/api/device/screen_all', { mode: 'on', serials: [_rec.serial] }); if (r && r.ok) await new Promise(function (z) { setTimeout(z, 800); }); } catch (e) { /* 唤醒失败也继续,用户自己会看到黑屏 */ } } // 取一张快照:截图 + 元素树(同一个连接背靠背取,避免框错位) async function recRefresh() { if (!_rec || !_rec.serial) return; const r = await apiGet('/api/uiauto/snapshot?serial=' + encodeURIComponent(_rec.serial)); if (!r || !r.ok) { const st = document.getElementById('rec-stage'); if (st) st.innerHTML = '
取画面失败:' + esc((r && r.error) || '未知错误') + '
'; return; } _rec.imgURL = r.image || ''; _rec.devW = r.width || 0; _rec.devH = r.height || 0; _rec.elements = r.elements || []; _rec.unstable = !!r.unstable; _rec.shots++; recPaint(); } function recPaint() { const st = document.getElementById('rec-stage'); if (!st) return; if (!_rec.imgURL) { st.innerHTML = '
正在取画面…
'; return; } st.innerHTML = '' + (_rec.unstable ? '
画面在变化,这次框可能不准
' : ''); const img = document.getElementById('rec-img'); img.addEventListener('mousedown', recDown); img.addEventListener('mousemove', recMove); img.addEventListener('mouseup', recUp); img.addEventListener('mouseleave', function () { _rec.dragFrom = null; recPaintOverlay(); }); recPaintOverlay(); } // 拖动中的轨迹提示(录制时能看见自己划到哪) function recPaintOverlay() { const st = document.getElementById('rec-stage'); if (!st) return; let cv = document.getElementById('rec-overlay-line'); if (!cv) { cv = document.createElement('div'); cv.id = 'rec-overlay-line'; cv.className = 'rec-line'; st.appendChild(cv); } if (!_rec.dragFrom) { cv.style.display = 'none'; return; } cv.style.display = 'block'; const a = _rec.dragFrom; const b = _rec.dragTo || a; const x = Math.min(a.x, b.x), y = Math.min(a.y, b.y); cv.style.left = x + 'px'; cv.style.top = y + 'px'; cv.style.width = Math.abs(b.x - a.x) + 'px'; cv.style.height = Math.abs(b.y - a.y) + 'px'; } function _recPos(e) { const img = document.getElementById('rec-img'); if (!img) return null; const r = img.getBoundingClientRect(); const x = e.clientX - r.left, y = e.clientY - r.top; if (x < 0 || y < 0 || x > r.width || y > r.height) return null; // 显示坐标 → 设备坐标 const sx = _rec.devW / r.width, sy = _rec.devH / r.height; return { x: Math.round(x * sx), y: Math.round(y * sy), rx: x, ry: y, cssW: r.width, cssH: r.height }; } function recDown(e) { if (_rec.busy) return; const p = _recPos(e); if (!p) return; _rec.dragFrom = p; _rec.dragTo = null; _rec.dragT0 = Date.now(); // 采样:拖动过程中把每个点连时间一起记下来(这就是"轨迹") _rec.samples = [[p.x, p.y, 0]]; recPaintOverlay(); } function recMove(e) { if (!_rec.dragFrom) return; const p = _recPos(e); if (!p) return; _rec.dragTo = p; const t = Date.now() - _rec.dragT0; const last = _rec.samples[_rec.samples.length - 1]; if (!last || t - last[2] >= 8 || Math.abs(p.x - last[0]) + Math.abs(p.y - last[1]) >= 3) { if (_rec.samples.length < 400) _rec.samples.push([p.x, p.y, t]); } recPaintOverlay(); } async function recUp(e) { if (!_rec.dragFrom || _rec.busy) { _rec.dragFrom = null; return; } const a = _rec.dragFrom; const b = _recPos(e) || a; const durMs = Date.now() - _rec.dragT0; _rec.dragFrom = null; recPaintOverlay(); const dist = Math.hypot(b.x - a.x, b.y - a.y); if (dist < 12 || durMs < 60) { // 视为点击 _rec.samples = []; await recTap(a.x, a.y); } else { const pts = _rec.samples.slice(); pts.push([b.x, b.y, durMs]); // 末点补上(决定手势收尾位置) _rec.samples = []; recPushGesture(pts, 'web'); await recRefresh(); } } // 一条录到的轨迹 → 「录制手势」步骤(**纯回放**:点列+时长原样存下,不再套滑动逻辑) function recPushGesture(points, src) { if (!points || points.length < 2) { showToast('没录到有效轨迹(太短了)', 'error'); return; } if (_rec.mode === 'gesture') { // 回填模式:直接交给调用方,收工 const cb = _rec.onDone; closeRecorder(); if (cb) cb({ points: points, speed: 1.0 }, src); return; } const dur = points[points.length - 1][2] || 0; _recPush({ type: 'gesture', label: '录制手势 ' + points.length + '点/' + (dur / 1000).toFixed(1) + 's', params: { points: points, speed: 1.0 } }); } // ---------- 手机端录制(getevent 抓真手指) ---------- async function recStartPhone() { if (!_rec.serial) { showToast('先选设备', 'error'); return; } const r = await apiPost('/api/gesture/record/start', { serial: _rec.serial }); if (!r || !r.ok) { showToast((r && r.error) || '开始失败', 'error'); return; } _rec.phone = { token: r.token, t0: Date.now() }; showToast('开始录制——现在用手指在手机上划', 'success'); recRenderFoot(); } async function recStopPhone() { if (!_rec.phone) return; const token = _rec.phone.token; _rec.phone = null; const r = await apiPost('/api/gesture/record/stop', { token: token }); if (!r || !r.ok) { showToast((r && r.error) || '停止失败', 'error'); recRenderFoot(); return; } const gs = r.gestures || []; if (!gs.length) { showToast('没录到手势——请在手机屏幕上用指划一下(不是点)', 'error'); recRenderFoot(); return; } if (_rec.mode === 'gesture') { recPushGesture(gs[gs.length - 1], 'phone'); return; } gs.forEach(function (g) { recPushGesture(g, 'phone'); }); showToast('录到 ' + gs.length + ' 个手势', 'success'); recRenderFoot(); recRefresh(); } // ---------- 翻译:点击 → 步骤 ---------- function _recFindElement(x, y) { // 取包含该点的最小元素;优先可点击的(和 /api/screen/tap 的 snap 同一口径) let best = null, bestClick = null; (_rec.elements || []).forEach(function (el) { const m = (el.bounds || '').match(/\[(\d+),(\d+)\]\[(\d+),(\d+)\]/); if (!m) return; const x1 = +m[1], y1 = +m[2], x2 = +m[3], y2 = +m[4]; if (!(x1 <= x && x <= x2 && y1 <= y && y <= y2)) return; const area = (x2 - x1) * (y2 - y1); const cand = { el: el, area: area, cx: Math.round((x1 + x2) / 2), cy: Math.round((y1 + y2) / 2) }; if (!best || area < best.area) best = cand; if (String(el.clickable) === 'true' && (!bestClick || area < bestClick.area)) bestClick = cand; }); return bestClick || best; } async function recTap(x, y) { if (!_rec || _rec.busy) return; _rec.busy = true; try { const hit = _recFindElement(x, y); let step; if (hit && hit.el.suggested && hit.el.suggested.value) { step = { type: 'click', label: '点击' + _recShort(hit.el), params: { selector_type: hit.el.suggested.type || 'xpath', selector_value: hit.el.suggested.value, wait_timeout: 2 } }; } else { step = { type: 'click_xy', label: '点击坐标', params: { x: Math.round(x / _rec.devW * 100), y: Math.round(y / _rec.devH * 100) } }; } _recPush(step); const r = await apiPost('/api/screen/tap', { serial: _rec.serial, x: x, y: y, snap: 1 }); if (r && !r.ok) showToast((r.error || '点击失败'), 'error'); await recRefresh(); } finally { _rec.busy = false; } } function _recShort(el) { const s = el.text || el.description || el.name || el.resource_id || ''; return s ? ('「' + String(s).slice(0, 12) + '」') : ''; } async function recKey(key) { if (!_rec || !_rec.serial) { showToast('先选设备', 'error'); return; } _recPush({ type: 'key_event', label: '按键 ' + key, params: { key: key } }); const r = await apiPost('/api/screen/key', { serial: _rec.serial, key: key }); if (r && !r.ok) showToast((r.error || '按键失败'), 'error'); await recRefresh(); } async function recText() { if (!_rec || !_rec.serial) { showToast('先选设备', 'error'); return; } const inp = document.getElementById('rec-text'); const t = (inp ? inp.value : '').trim(); if (!t) { showToast('先填要输入的文字', 'error'); return; } _recPush({ type: 'input_text', label: '输入 ' + t.slice(0, 8), params: { mode: 'fixed', fixed_text: t, clear_first: false } }); const r = await apiPost('/api/screen/text', { serial: _rec.serial, text: t }); if (r && !r.ok) showToast((r.error || '输入失败'), 'error'); if (inp) inp.value = ''; await recRefresh(); } function recAddWait() { _recPush({ type: 'wait', label: '等待', params: { min: 1, max: 3 } }); } // 入列:按需自动补一条「等待」(真实停顿,±15%),并刷新列表 function _recPush(step) { const now = Date.now(); if (_rec.keepPause && _rec.lastTs && step.type !== 'wait') { const gap = (now - _rec.lastTs) / 1000; if (gap >= 1.0) { _rec.steps.push({ id: 'rec_' + Math.random().toString(36).slice(2, 8), type: 'wait', label: '等待 ' + gap.toFixed(1) + 's', params: { min: +(gap * 0.85).toFixed(1), max: +(gap * 1.15).toFixed(1) } }); } } step.id = 'rec_' + Math.random().toString(36).slice(2, 8); _rec.steps.push(step); _rec.lastTs = now; recRender(); } function recClear() { _rec.steps = []; _rec.lastTs = 0; recRender(); } function recDel(i) { _rec.steps.splice(i, 1); recRender(); } function recMove2(i, d) { const j = i + d; if (j < 0 || j >= _rec.steps.length) return; const t = _rec.steps[i]; _rec.steps[i] = _rec.steps[j]; _rec.steps[j] = t; recRender(); } function recRender() { const list = document.getElementById('rec-list'); if (!list) return; const steps = _rec ? _rec.steps : []; document.getElementById('rec-count').textContent = steps.length ? ('共 ' + steps.length + ' 步') : ''; list.innerHTML = steps.map(function (s, i) { return '
' + (i + 1) + '' + '' + esc(s.label || s.type) + '' + '' + '' + '' + '' + '
'; }).join('') || '
还没有录到步骤——在左边画面上点一下或划一下试试
'; recRenderFoot(); } function recRenderFoot() { const foot = document.getElementById('rec-foot'); if (!foot) return; const steps = _rec.steps; if (_rec.mode === 'gesture') { // 回填模式:录到就立刻填回那个步骤,不用再点确认 foot.innerHTML = _rec.phone ? '' : '' + '
或在左边画面上按住鼠标拖一下——录到就立刻填进那一步
'; return; } const phoneBtn = _rec.phone ? '' : ''; foot.innerHTML = '
' + phoneBtn + '' + (_rec.phone ? '正在录:请用手指在手机屏幕上划,划完点「停止」' : '手机上用手指划 = 录真手指轨迹;在左边画面上拖 = 网页录制') + '
' + '
' + '
' + '
划出来的每一段都是一条「录制手势」步骤(按原路径与时长回放,不再套滑动逻辑)
'; } async function recSaveAsAction() { const el = document.getElementById('rec-name'); const name = (el ? el.value : '').trim(); if (!name) { showToast('先给动作起个名', 'error'); return; } const steps = _rec.steps.filter(function (s) { return s.type !== 'wait' || true; }) .map(function (s) { return { type: s.type, label: s.label, params: s.params }; }); if (!steps.length) { showToast('还没有录到步骤', 'error'); return; } const r = await apiPost('/api/custom_actions', { name: name, icon: '⏺', steps: steps }); if (!r || !r.ok) { showToast((r && r.error) || '保存失败', 'error'); return; } showToast('已存成动作「' + name + '」(' + steps.length + ' 步)', 'success'); if (typeof loadCustomActions === 'function') loadCustomActions(); closeRecorder(); }