// ================== 动作录制器 ================== // 在设备截图上操作(点/划/按键/输入),自动翻译成**步骤**: // · 点 → 命中元素就记「点击元素」(选择器),抓不到就退化成「点击坐标」(百分比) // · 划 → 记「滑动」:方向 + 幅度(屏占比) + 实际时长 // · 停顿 → 可选记成「等待」(按真实间隔 ±15%) // // 两种用法(同一个录窗口): // mode='action' —— 录一段,存成「自定义动作」(进动作库,能拖进任何任务) // mode='gesture' —— 只取最后一个手势,回填当前滑动步骤的参数 // // 元素树用 /api/uiauto/snapshot 一次取齐(截图 + 元素),**每次操作后自动刷新**, // 这样下一次点击就能用最新的树翻译选择器(抖音这种没有 id 的界面靠 text/desc)。 let _rec = null; function _recEl() { return document.getElementById('rec-overlay'); } async function openRecorder(opts) { opts = opts || {}; if (_recEl()) return; _rec = { mode: opts.mode || 'action', serial: opts.serial || '', onDone: opts.onDone || null, steps: [], elements: [], devW: 0, devH: 0, imgURL: '', lastTs: 0, dragFrom: null, busy: false, keepPause: true, shots: 0, unstable: false, }; const ov = document.createElement('div'); ov.id = 'rec-overlay'; ov.className = 'modal-overlay show'; // .modal-overlay 默认 display:none,必须带 show ov.innerHTML = ''; ov.addEventListener('mousedown', function (e) { if (e.target === ov) closeRecorder(); }); document.body.appendChild(ov); await recLoadDevices(); recRender(); } function closeRecorder() { const ov = _recEl(); if (ov) ov.remove(); _rec = null; } async function recLoadDevices() { const sel = document.getElementById('rec-device'); if (!sel) return; sel.innerHTML = ''; const r = await apiGet('/api/devices'); if (!r || !r.ok) return; (r.items || []).forEach(function (d) { const o = document.createElement('option'); o.value = d.serial; o.textContent = d.name ? (d.name + '(' + d.serial + ')') : d.serial; sel.appendChild(o); }); if (_rec.serial) { sel.value = _rec.serial; } else if ((r.items || []).length) { sel.value = r.items[0].serial; _rec.serial = r.items[0].serial; } if (_rec.serial) recRefresh(); } function recPickDevice() { const sel = document.getElementById('rec-device'); _rec.serial = sel ? sel.value : ''; _rec.elements = []; recRender(); if (_rec.serial) recRefresh(); } // 取一张快照:截图 + 元素树(同一个连接背靠背取,避免框错位) async function recRefresh() { if (!_rec || !_rec.serial) return; const r = await apiGet('/api/uiauto/snapshot?serial=' + encodeURIComponent(_rec.serial)); if (!r || !r.ok) { const st = document.getElementById('rec-stage'); if (st) st.innerHTML = '
取画面失败:' + esc((r && r.error) || '未知错误') + '
'; return; } _rec.imgURL = r.image || ''; _rec.devW = r.width || 0; _rec.devH = r.height || 0; _rec.elements = r.elements || []; _rec.unstable = !!r.unstable; _rec.shots++; recPaint(); } function recPaint() { const st = document.getElementById('rec-stage'); if (!st) return; if (!_rec.imgURL) { st.innerHTML = '
正在取画面…
'; return; } st.innerHTML = '' + (_rec.unstable ? '
画面在变化,这次框可能不准
' : ''); const img = document.getElementById('rec-img'); img.addEventListener('mousedown', recDown); img.addEventListener('mousemove', recMove); img.addEventListener('mouseup', recUp); img.addEventListener('mouseleave', function () { _rec.dragFrom = null; recPaintOverlay(); }); recPaintOverlay(); } // 拖动中的轨迹提示(录制时能看见自己划到哪) function recPaintOverlay() { const st = document.getElementById('rec-stage'); if (!st) return; let cv = document.getElementById('rec-overlay-line'); if (!cv) { cv = document.createElement('div'); cv.id = 'rec-overlay-line'; cv.className = 'rec-line'; st.appendChild(cv); } if (!_rec.dragFrom) { cv.style.display = 'none'; return; } cv.style.display = 'block'; const a = _rec.dragFrom; const b = _rec.dragTo || a; const x = Math.min(a.x, b.x), y = Math.min(a.y, b.y); cv.style.left = x + 'px'; cv.style.top = y + 'px'; cv.style.width = Math.abs(b.x - a.x) + 'px'; cv.style.height = Math.abs(b.y - a.y) + 'px'; } function _recPos(e) { const img = document.getElementById('rec-img'); if (!img) return null; const r = img.getBoundingClientRect(); const x = e.clientX - r.left, y = e.clientY - r.top; if (x < 0 || y < 0 || x > r.width || y > r.height) return null; // 显示坐标 → 设备坐标 const sx = _rec.devW / r.width, sy = _rec.devH / r.height; return { x: Math.round(x * sx), y: Math.round(y * sy), rx: x, ry: y, cssW: r.width, cssH: r.height }; } function recDown(e) { if (_rec.busy) return; const p = _recPos(e); if (!p) return; _rec.dragFrom = p; _rec.dragTo = null; _rec.dragT0 = Date.now(); recPaintOverlay(); } function recMove(e) { if (!_rec.dragFrom) return; const p = _recPos(e); if (!p) return; _rec.dragTo = p; recPaintOverlay(); } async function recUp(e) { if (!_rec.dragFrom || _rec.busy) { _rec.dragFrom = null; return; } const a = _rec.dragFrom; const b = _recPos(e) || a; const durMs = Date.now() - _rec.dragT0; _rec.dragFrom = null; recPaintOverlay(); const dist = Math.hypot(b.x - a.x, b.y - a.y); if (dist < 12 || durMs < 60) { // 视为点击 await recTap(a.x, a.y); } else { await recSwipe(a, b, durMs); } } // ---------- 翻译:点击 → 步骤 ---------- function _recFindElement(x, y) { // 取包含该点的最小元素;优先可点击的(和 /api/screen/tap 的 snap 同一口径) let best = null, bestClick = null; (_rec.elements || []).forEach(function (el) { const m = (el.bounds || '').match(/\[(\d+),(\d+)\]\[(\d+),(\d+)\]/); if (!m) return; const x1 = +m[1], y1 = +m[2], x2 = +m[3], y2 = +m[4]; if (!(x1 <= x && x <= x2 && y1 <= y && y <= y2)) return; const area = (x2 - x1) * (y2 - y1); const cand = { el: el, area: area, cx: Math.round((x1 + x2) / 2), cy: Math.round((y1 + y2) / 2) }; if (!best || area < best.area) best = cand; if (String(el.clickable) === 'true' && (!bestClick || area < bestClick.area)) bestClick = cand; }); return bestClick || best; } async function recTap(x, y) { if (!_rec || _rec.busy) return; _rec.busy = true; try { const hit = _recFindElement(x, y); let step; if (hit && hit.el.suggested && hit.el.suggested.value) { step = { type: 'click', label: '点击' + _recShort(hit.el), params: { selector_type: hit.el.suggested.type || 'xpath', selector_value: hit.el.suggested.value, wait_timeout: 2 } }; } else { step = { type: 'click_xy', label: '点击坐标', params: { x: Math.round(x / _rec.devW * 100), y: Math.round(y / _rec.devH * 100) } }; } _recPush(step); const r = await apiPost('/api/screen/tap', { serial: _rec.serial, x: x, y: y, snap: 1 }); if (r && !r.ok) showToast((r.error || '点击失败'), 'error'); await recRefresh(); } finally { _rec.busy = false; } } function _recShort(el) { const s = el.text || el.description || el.name || el.resource_id || ''; return s ? ('「' + String(s).slice(0, 12) + '」') : ''; } // ---------- 翻译:滑动 → 步骤 ---------- async function recSwipe(a, b, durMs) { if (!_rec || _rec.busy) return; _rec.busy = true; try { const dx = b.x - a.x, dy = b.y - a.y; const vertical = Math.abs(dy) >= Math.abs(dx); const direction = vertical ? (dy < 0 ? 'up' : 'down') : (dx < 0 ? 'left' : 'right'); const span = Math.abs(vertical ? dy : dx); const base = vertical ? _rec.devH : _rec.devW; const ratio = Math.max(0.05, Math.min(0.9, span / base)); const dur = Math.max(0.08, durMs / 1000); const step = { type: 'swipe', label: '滑动 ' + direction, params: { direction: direction, duration_min: +(dur * 0.9).toFixed(2), duration_max: +(dur * 1.15).toFixed(2), distance_ratio: +ratio.toFixed(2), jitter: 0.15, humanize: true } }; _recPush(step); const r = await apiPost('/api/screen/swipe', { serial: _rec.serial, x1: a.x, y1: a.y, x2: b.x, y2: b.y, duration: Math.round(durMs) / 1000 }); if (r && !r.ok) showToast((r.error || '滑动失败'), 'error'); await recRefresh(); } finally { _rec.busy = false; } } async function recKey(key) { if (!_rec || !_rec.serial) { showToast('先选设备', 'error'); return; } _recPush({ type: 'key_event', label: '按键 ' + key, params: { key: key } }); const r = await apiPost('/api/screen/key', { serial: _rec.serial, key: key }); if (r && !r.ok) showToast((r.error || '按键失败'), 'error'); await recRefresh(); } async function recText() { if (!_rec || !_rec.serial) { showToast('先选设备', 'error'); return; } const inp = document.getElementById('rec-text'); const t = (inp ? inp.value : '').trim(); if (!t) { showToast('先填要输入的文字', 'error'); return; } _recPush({ type: 'input_text', label: '输入 ' + t.slice(0, 8), params: { mode: 'fixed', fixed_text: t, clear_first: false } }); const r = await apiPost('/api/screen/text', { serial: _rec.serial, text: t }); if (r && !r.ok) showToast((r.error || '输入失败'), 'error'); if (inp) inp.value = ''; await recRefresh(); } function recAddWait() { _recPush({ type: 'wait', label: '等待', params: { min: 1, max: 3 } }); } // 入列:按需自动补一条「等待」(真实停顿,±15%),并刷新列表 function _recPush(step) { const now = Date.now(); if (_rec.keepPause && _rec.lastTs && step.type !== 'wait') { const gap = (now - _rec.lastTs) / 1000; if (gap >= 1.0) { _rec.steps.push({ id: 'rec_' + Math.random().toString(36).slice(2, 8), type: 'wait', label: '等待 ' + gap.toFixed(1) + 's', params: { min: +(gap * 0.85).toFixed(1), max: +(gap * 1.15).toFixed(1) } }); } } step.id = 'rec_' + Math.random().toString(36).slice(2, 8); _rec.steps.push(step); _rec.lastTs = now; recRender(); } function recClear() { _rec.steps = []; _rec.lastTs = 0; recRender(); } function recDel(i) { _rec.steps.splice(i, 1); recRender(); } function recMove2(i, d) { const j = i + d; if (j < 0 || j >= _rec.steps.length) return; const t = _rec.steps[i]; _rec.steps[i] = _rec.steps[j]; _rec.steps[j] = t; recRender(); } function recRender() { const list = document.getElementById('rec-list'); if (!list) return; const steps = _rec ? _rec.steps : []; document.getElementById('rec-count').textContent = steps.length ? ('共 ' + steps.length + ' 步') : ''; list.innerHTML = steps.map(function (s, i) { return '
' + (i + 1) + '' + '' + esc(s.label || s.type) + '' + '' + '' + '' + '' + '
'; }).join('') || '
还没有录到步骤——在左边画面上点一下或划一下试试
'; recRenderFoot(); } function recRenderFoot() { const foot = document.getElementById('rec-foot'); if (!foot) return; const steps = _rec.steps; if (_rec.mode === 'gesture') { const sw = steps.filter(function (s) { return s.type === 'swipe'; }); foot.innerHTML = sw.length ? '' + '
把方向/幅度/时长回填到当前滑动步骤
' : '
在画面上划一下(按住拖动),然后点这里
'; return; } foot.innerHTML = '
' + '
' + '
存进「自定义动作」库,之后能拖进任何任务的步骤画布
'; } function recApplyGesture() { const sw = _rec.steps.filter(function (s) { return s.type === 'swipe'; }); if (!sw.length) return; const p = sw[sw.length - 1].params; const cb = _rec.onDone; closeRecorder(); if (cb) cb(p); } async function recSaveAsAction() { const el = document.getElementById('rec-name'); const name = (el ? el.value : '').trim(); if (!name) { showToast('先给动作起个名', 'error'); return; } const steps = _rec.steps.filter(function (s) { return s.type !== 'wait' || true; }) .map(function (s) { return { type: s.type, label: s.label, params: s.params }; }); if (!steps.length) { showToast('还没有录到步骤', 'error'); return; } const r = await apiPost('/api/custom_actions', { name: name, icon: '⏺', steps: steps }); if (!r || !r.ok) { showToast((r && r.error) || '保存失败', 'error'); return; } showToast('已存成动作「' + name + '」(' + steps.length + ' 步)', 'success'); if (typeof loadCustomActions === 'function') loadCustomActions(); closeRecorder(); }