feat(monitor): 抖音 Web 接口客户端 —— 绕开爬虫子进程,直接发 HTTP
移植自 mac-agent-os 的 mediacrawler_adapter:不起子进程、不开页面,用浏览器里那份
登录态直接调抖音 Web 接口。产物键名照抄 store/douyin,所以 ingest 那条链路一个字不用改。
**目前能用的(真环境实测,非推断)**:
profile/other : 200, 7075 字节 —— 博主主页指标(粉丝/获赞/作品数/昵称)
aweme/detail : 200, 45425 字节 —— 单条作品详情(含点赞/评论/收藏/分享)
**目前不能用的:作品列表 `aweme/post`。** 两个互相独立的原因:
1. 这个接口被抖音单独升级成了真校验:不带 x-tt-argus 回 403「Uifid Not Found」,
带上 dummy 值回 200 + **空 body**。也就是说「头在不在」骗得过,「真校验」过不了。
同一套头打 profile/other 和 aweme/detail 都是通的 —— 抖音是挑着接口加保护的,
挑中的恰好是「批量拉作品列表」这个最敏感的动作。
2. 改走页面截获也不行:CDP 浏览器打开博主主页会落到「验证码中间页」(当天大量探测的
代价,过几小时要重测)。
所以现在的边界是:**已知作品的指标刷新能做,自动发现新作品做不了**。
**排查中控住变量后得到的两条事实**(都写进注释了):
· `Accept` / `Accept-Language` / `Referer` 才是主页接口能返回真数据的原因 —— 只有
UA+client hints+Cookie 时是 200 但仅 121 字节的空壳,补上这三个头变 7074 字节。
(我先前猜的 sec-ch-ua 不是关键。)
· 因此 UA 与 client hints 必须**成套地取自同一个浏览器**,所以 BrowserIdentity 一次
从 CDP 取齐 cookie + UA + hints,而不是各自写死。
「200 + 空 body 必须当场报错」也是刻意写死的:放过去它会在下游变成「这个博主没作品」,
把一次失败伪装成一条正常结果 —— 爬虫那条路正是这么栽的,还被翻译成「账号被封」。
测试 +11:cookie 解析、请求头成套性(含 uifid 缺失/回退)、产物键名与 store 对齐、
以及 _get 的三条失败路径(空 body / 403 带网关原话 / 正常返回)。
This commit is contained in:
@@ -0,0 +1 @@
|
||||
# nurture 模块
|
||||
@@ -0,0 +1,242 @@
|
||||
"""
|
||||
anchor_db.py — 锚点规则数据库管理
|
||||
|
||||
锚点规则 + 原子操作 + 检测日志 + 校准候选 全部存储在 SQLite 中。
|
||||
可拆解、可更新、可版本管理。
|
||||
"""
|
||||
import json
|
||||
import sqlite3
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
DB_DIR = Path(__file__).parent / "data"
|
||||
DB_PATH = DB_DIR / "anchor.db"
|
||||
SCHEMA_PATH = DB_DIR / "schema.sql"
|
||||
|
||||
|
||||
# ── 数据库初始化 ──
|
||||
|
||||
def get_conn() -> sqlite3.Connection:
|
||||
"""获取数据库连接(每次调用创建新连接,线程安全)"""
|
||||
DB_DIR.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
return conn
|
||||
|
||||
|
||||
def init_db(force: bool = False):
|
||||
"""初始化/重置数据库"""
|
||||
if force and DB_PATH.exists():
|
||||
DB_PATH.unlink()
|
||||
conn = get_conn()
|
||||
# 检查是否已初始化
|
||||
cursor = conn.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='anchor_rules'")
|
||||
if cursor.fetchone() and not force:
|
||||
conn.close()
|
||||
return
|
||||
|
||||
# 执行 schema
|
||||
schema = SCHEMA_PATH.read_text(encoding='utf-8')
|
||||
conn.executescript(schema)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"✅ 数据库已初始化: {DB_PATH}")
|
||||
|
||||
|
||||
# ── 锚点规则 CRUD ──
|
||||
|
||||
def get_rules(state_name: str = None) -> list:
|
||||
"""获取锚点规则
|
||||
|
||||
Args:
|
||||
state_name: 指定状态名, None=返回所有
|
||||
Returns:
|
||||
[{state_name, field, operator, value, logic_group}, ...]
|
||||
"""
|
||||
conn = get_conn()
|
||||
if state_name:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM anchor_rules WHERE state_name=? AND active=1 ORDER BY logic_group, id",
|
||||
(state_name,)
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM anchor_rules WHERE active=1 ORDER BY state_name, logic_group, id"
|
||||
).fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def add_rule(state_name: str, field: str, operator: str, value: str,
|
||||
logic_group: int = 0) -> int:
|
||||
"""添加锚点规则"""
|
||||
conn = get_conn()
|
||||
cursor = conn.execute(
|
||||
"INSERT INTO anchor_rules (state_name, field, operator, value, logic_group) VALUES (?,?,?,?,?)",
|
||||
(state_name, field, operator, value, logic_group)
|
||||
)
|
||||
conn.commit()
|
||||
new_id = cursor.lastrowid
|
||||
conn.close()
|
||||
return new_id
|
||||
|
||||
|
||||
def update_rule(rule_id: int, **kwargs):
|
||||
"""更新规则字段"""
|
||||
allowed = {'field', 'operator', 'value', 'logic_group', 'active'}
|
||||
sets = []
|
||||
params = []
|
||||
for k, v in kwargs.items():
|
||||
if k in allowed:
|
||||
sets.append(f"{k}=?")
|
||||
params.append(v)
|
||||
if not sets:
|
||||
return
|
||||
params.append(rule_id)
|
||||
conn = get_conn()
|
||||
conn.execute(f"UPDATE anchor_rules SET {', '.join(sets)}, updated_at=datetime('now','localtime') WHERE id=?", params)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def delete_rule(rule_id: int):
|
||||
"""软删除规则"""
|
||||
update_rule(rule_id, active=0)
|
||||
|
||||
|
||||
# ── 原子操作 CRUD ──
|
||||
|
||||
def get_ops(op_name: str = None) -> list:
|
||||
"""获取原子操作"""
|
||||
conn = get_conn()
|
||||
if op_name:
|
||||
rows = conn.execute("SELECT * FROM atomic_ops WHERE op_name=?", (op_name,)).fetchall()
|
||||
else:
|
||||
rows = conn.execute("SELECT * FROM atomic_ops ORDER BY id").fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def add_op(op_name: str, description: str, action_type: str,
|
||||
target: str, verify_rule: str = None) -> int:
|
||||
"""添加原子操作"""
|
||||
conn = get_conn()
|
||||
cursor = conn.execute(
|
||||
"INSERT INTO atomic_ops (op_name, description, action_type, target, verify_rule) VALUES (?,?,?,?,?)",
|
||||
(op_name, description, action_type, target, verify_rule)
|
||||
)
|
||||
conn.commit()
|
||||
new_id = cursor.lastrowid
|
||||
conn.close()
|
||||
return new_id
|
||||
|
||||
|
||||
# ── 检测日志 ──
|
||||
|
||||
def log_detection(identity: str, expected_state: str, detected_state: str,
|
||||
match: bool, anchor_snapshot: dict = None,
|
||||
screenshot_path: str = None):
|
||||
"""记录一次锚点检测结果"""
|
||||
conn = get_conn()
|
||||
conn.execute(
|
||||
"INSERT INTO detection_logs (identity, expected_state, detected_state, match, anchor_snapshot, screenshot_path) VALUES (?,?,?,?,?,?)",
|
||||
(identity, expected_state, detected_state, int(match),
|
||||
json.dumps(anchor_snapshot, ensure_ascii=False) if anchor_snapshot else None,
|
||||
screenshot_path)
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_recent_accuracy(identity: str = None, count: int = 50) -> float:
|
||||
"""获取最近 N 次检测的成功率"""
|
||||
conn = get_conn()
|
||||
if identity:
|
||||
rows = conn.execute(
|
||||
"SELECT match FROM detection_logs WHERE identity=? ORDER BY id DESC LIMIT ?",
|
||||
(identity, count)
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT match FROM detection_logs ORDER BY id DESC LIMIT ?",
|
||||
(count,)
|
||||
).fetchall()
|
||||
conn.close()
|
||||
if not rows:
|
||||
return 1.0
|
||||
return sum(r['match'] for r in rows) / len(rows)
|
||||
|
||||
|
||||
def get_recent_failures(identity: str = None, threshold: int = 5) -> bool:
|
||||
"""检查最近是否连续失败 threshold 次"""
|
||||
conn = get_conn()
|
||||
if identity:
|
||||
rows = conn.execute(
|
||||
"SELECT match FROM detection_logs WHERE identity=? ORDER BY id DESC LIMIT ?",
|
||||
(identity, threshold)
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT match FROM detection_logs ORDER BY id DESC LIMIT ?",
|
||||
(threshold,)
|
||||
).fetchall()
|
||||
conn.close()
|
||||
if len(rows) < threshold:
|
||||
return False
|
||||
return all(r['match'] == 0 for r in rows)
|
||||
|
||||
|
||||
# ── 校准候选 ──
|
||||
|
||||
def add_candidate(state_name: str, source: str, old_rules: list, new_rules: list) -> int:
|
||||
"""添加校准候选"""
|
||||
conn = get_conn()
|
||||
cursor = conn.execute(
|
||||
"INSERT INTO calibration_candidates (state_name, source, old_rules, new_rules) VALUES (?,?,?,?)",
|
||||
(state_name, source, json.dumps(old_rules, ensure_ascii=False),
|
||||
json.dumps(new_rules, ensure_ascii=False))
|
||||
)
|
||||
conn.commit()
|
||||
new_id = cursor.lastrowid
|
||||
conn.close()
|
||||
return new_id
|
||||
|
||||
|
||||
def update_candidate(candidate_id: int, status: str, test_count: int = None, test_pass: int = None):
|
||||
"""更新校准候选状态"""
|
||||
conn = get_conn()
|
||||
sets = ["status=?"]
|
||||
params = [status]
|
||||
if test_count is not None:
|
||||
sets.append("test_count=?")
|
||||
params.append(test_count)
|
||||
if test_pass is not None:
|
||||
sets.append("test_pass=?")
|
||||
params.append(test_pass)
|
||||
params.append(candidate_id)
|
||||
conn.execute(f"UPDATE calibration_candidates SET {', '.join(sets)} WHERE id=?", params)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
# ── 状态迁移 ──
|
||||
|
||||
def get_valid_next_states(from_state: str) -> list:
|
||||
"""获取从当前状态可以迁移到哪些状态"""
|
||||
conn = get_conn()
|
||||
rows = conn.execute(
|
||||
"SELECT to_state, via_op FROM state_transitions WHERE from_state=?",
|
||||
(from_state,)
|
||||
).fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 初始化数据库并导入初始数据
|
||||
init_db(force=True)
|
||||
print(f" 规则数: {len(get_rules())}")
|
||||
print(f" 操作数: {len(get_ops())}")
|
||||
@@ -0,0 +1,222 @@
|
||||
"""
|
||||
anchor_rules.py — 锚点检测引擎
|
||||
|
||||
从 anchor.db 读取规则,执行状态检测。
|
||||
规则可动态更新(通过 anchor_db.py),无需修改代码。
|
||||
"""
|
||||
import re
|
||||
import json
|
||||
from . import anchor_db as db
|
||||
|
||||
|
||||
# ── 运算符映射 ──
|
||||
|
||||
def _apply_op(field_value, operator, target_value):
|
||||
"""对单个字段应用运算符比较(不区分大小写)"""
|
||||
if field_value is None:
|
||||
return False
|
||||
|
||||
# 全部转字符串,不区分大小写
|
||||
fv = str(field_value).lower()
|
||||
tv = str(target_value).lower()
|
||||
|
||||
if operator == 'eq':
|
||||
return fv == tv
|
||||
elif operator == 'neq':
|
||||
return fv != tv
|
||||
elif operator == 'gt':
|
||||
try:
|
||||
return float(fv) > float(tv)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
elif operator == 'gte':
|
||||
try:
|
||||
return float(fv) >= float(tv)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
elif operator == 'lt':
|
||||
try:
|
||||
return float(fv) < float(tv)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
elif operator == 'lte':
|
||||
try:
|
||||
return float(fv) <= float(tv)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
elif operator == 'contains':
|
||||
return tv in fv
|
||||
elif operator == 'not_contains':
|
||||
return tv not in fv
|
||||
elif operator == 'regex':
|
||||
try:
|
||||
return bool(re.search(tv, fv))
|
||||
except re.error:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
# ── 核心检测逻辑 ──
|
||||
|
||||
def detect_state(anchors: dict, strict: bool = False) -> str:
|
||||
"""检测当前页面状态
|
||||
|
||||
Args:
|
||||
anchors: DOM 锚点数据字典
|
||||
strict: True=必须所有规则满足; False=按置信度选最佳匹配
|
||||
|
||||
Returns:
|
||||
state_name 或 'UNKNOWN'
|
||||
"""
|
||||
rules = db.get_rules()
|
||||
if not rules:
|
||||
return "UNKNOWN"
|
||||
|
||||
# 按 state_name 分组
|
||||
state_rules = {}
|
||||
for r in rules:
|
||||
sn = r['state_name']
|
||||
if sn not in state_rules:
|
||||
state_rules[sn] = {}
|
||||
lg = r['logic_group']
|
||||
if lg not in state_rules[sn]:
|
||||
state_rules[sn][lg] = []
|
||||
state_rules[sn][lg].append(r)
|
||||
|
||||
best_state = "UNKNOWN"
|
||||
best_score = 0
|
||||
|
||||
# 检测优先级(子状态优先于父状态)
|
||||
PRIORITY = ['SEARCH_PLAYER', 'DETAIL', 'PLAYER', 'SEARCH', 'HOME', 'POPUP', 'LOGIN']
|
||||
state_groups = sorted(state_rules.items(),
|
||||
key=lambda x: PRIORITY.index(x[0]) if x[0] in PRIORITY else 99)
|
||||
|
||||
for state_name, groups in state_groups:
|
||||
# 每个 logic_group 内的规则 AND,不同 group 之间 OR
|
||||
group_results = []
|
||||
for group_id, group_rules in groups.items():
|
||||
all_match = all(
|
||||
_apply_op(anchors.get(r['field']), r['operator'], r['value'])
|
||||
for r in group_rules
|
||||
)
|
||||
group_results.append(all_match)
|
||||
|
||||
# OR 连接:任一 group 通过即该状态匹配
|
||||
state_match = any(group_results)
|
||||
|
||||
if strict:
|
||||
if state_match:
|
||||
return state_name
|
||||
else:
|
||||
# 非严格模式:计算匹配得分
|
||||
total_rules = sum(len(g) for g in groups.values())
|
||||
matched_rules = sum(
|
||||
sum(1 for r in g if _apply_op(anchors.get(r['field']), r['operator'], r['value']))
|
||||
for g in groups.values()
|
||||
)
|
||||
score = matched_rules / max(total_rules, 1)
|
||||
if state_match and score > best_score:
|
||||
best_score = score
|
||||
best_state = state_name
|
||||
elif not state_match and score > best_score:
|
||||
# 即使不完全匹配,高分也记录
|
||||
if score > 0.6:
|
||||
best_score = score
|
||||
best_state = f"{state_name}?"
|
||||
|
||||
return best_state
|
||||
|
||||
|
||||
def verify_op(op_name: str, anchors: dict) -> bool:
|
||||
"""验证原子操作是否成功执行"""
|
||||
ops = db.get_ops(op_name)
|
||||
if not ops:
|
||||
return True # 无验证规则默认通过
|
||||
op = ops[0]
|
||||
verify = op.get('verify_rule')
|
||||
if not verify:
|
||||
return True
|
||||
|
||||
# 验证规则映射
|
||||
VERIFY_MAP = {
|
||||
'video_count_gte_3': lambda a: a.get('video_count', 0) >= 3,
|
||||
'video_src_change': lambda a: True, # 需要前后对比
|
||||
'like_count_change': lambda a: True,
|
||||
'title_contains_搜索': lambda a: '搜索' in str(a.get('title', '')),
|
||||
'video_count_eq_1': lambda a: a.get('video_count', 0) == 1,
|
||||
'popup_gone': lambda a: not a.get('has_popup_close', False),
|
||||
}
|
||||
|
||||
check = VERIFY_MAP.get(verify)
|
||||
if check:
|
||||
return check(anchors)
|
||||
return True
|
||||
|
||||
|
||||
def log_and_check(identity: str, expected_state: str, anchors: dict,
|
||||
screenshot_path: str = None) -> dict:
|
||||
"""检测 + 记录 + 返回结果"""
|
||||
detected = detect_state(anchors)
|
||||
match = (detected == expected_state)
|
||||
|
||||
db.log_detection(
|
||||
identity=identity,
|
||||
expected_state=expected_state,
|
||||
detected_state=detected,
|
||||
match=match,
|
||||
anchor_snapshot=anchors,
|
||||
screenshot_path=screenshot_path,
|
||||
)
|
||||
|
||||
return {
|
||||
"expected": expected_state,
|
||||
"detected": detected,
|
||||
"match": match,
|
||||
"accuracy": db.get_recent_accuracy(identity),
|
||||
"needs_calibration": db.get_recent_failures(identity, threshold=5),
|
||||
}
|
||||
|
||||
|
||||
# ── 便捷函数 ──
|
||||
|
||||
def list_rules(state_name: str = None) -> str:
|
||||
"""以人类可读格式列出规则"""
|
||||
rules = db.get_rules(state_name)
|
||||
if not rules:
|
||||
return "暂无锚点规则"
|
||||
|
||||
lines = []
|
||||
current_state = None
|
||||
for r in rules:
|
||||
if r['state_name'] != current_state:
|
||||
current_state = r['state_name']
|
||||
lines.append(f"\n[{current_state}]")
|
||||
lines.append(f" {r['field']} {r['operator']} {r['value']} (group={r['logic_group']})")
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def list_ops() -> str:
|
||||
"""列出所有原子操作"""
|
||||
ops = db.get_ops()
|
||||
if not ops:
|
||||
return "暂无原子操作"
|
||||
lines = []
|
||||
for op in ops:
|
||||
lines.append(f" {op['op_name']:15s} | {op['action_type']:10s} | {op.get('target', ''):30s} | {op.get('verify_rule', '-')}")
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 测试:用采集数据验证规则
|
||||
import json
|
||||
summary_path = db.DB_DIR.parent.parent / "data" / "anchor_summary.json"
|
||||
if summary_path.exists():
|
||||
data = json.loads(summary_path.read_text())
|
||||
print("📊 用采集数据验证锚点规则:")
|
||||
for d in data:
|
||||
step = d['step']
|
||||
anchors = d['anchors']
|
||||
state = detect_state(anchors)
|
||||
print(f" 步{step}: {state:15s} | {d['url'][:40]}")
|
||||
print(f"\n📋 当前规则:\n{list_rules()}")
|
||||
print(f"\n📋 原子操作:\n{list_ops()}")
|
||||
@@ -0,0 +1,187 @@
|
||||
"""
|
||||
行为模拟参数配置 — 拟人化操作行为
|
||||
|
||||
每个身份可以独立配置 behavior 参数,覆盖默认值。
|
||||
默认配置在 DEFAULT_CONFIG 中定义,用户可在 identities/{name}/config.yaml 中覆盖。
|
||||
"""
|
||||
import random
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
# ── 默认行为配置 ──────────────────────────────────────────────
|
||||
|
||||
DEFAULT_CONFIG = {
|
||||
# 基础延迟
|
||||
'base_delay': 1.5, # 操作间基础间隔(秒)
|
||||
'delay_variance': 0.8, # 随机波动范围
|
||||
|
||||
# 鼠标行为
|
||||
'mouse': {
|
||||
'speed_range': [0.5, 1.5], # 鼠标移动速度系数
|
||||
'click_delay': [0.08, 0.25], # 点击前停顿(秒)
|
||||
},
|
||||
|
||||
# 滚动行为
|
||||
'scroll': {
|
||||
'mode': 'human', # human / random / fast
|
||||
'pause_chance': 0.3, # 滚动中暂停的概率
|
||||
'pause_duration': [0.5, 2.0], # 暂停时长范围(秒)
|
||||
'distance_range': [200, 700], # 每次滚动距离(px)
|
||||
},
|
||||
|
||||
# 操作顺序
|
||||
'sequence': {
|
||||
'shuffle': True, # 是否打乱操作顺序
|
||||
'repetition_chance': 0.1, # 重复上一个操作的概率
|
||||
'min_gap': 1.5, # 操作间最小间隔(秒)
|
||||
'max_gap': 6.0, # 操作间最大间隔(秒)
|
||||
},
|
||||
|
||||
# 注意力模拟
|
||||
'attention': {
|
||||
'distraction_chance': 0.05, # "分心"概率
|
||||
'distraction_duration': [2, 6], # 分心持续时长(秒)
|
||||
'watch_duration': [4, 12], # 观看视频时长(秒)
|
||||
'linger_after_action': [1, 4], # 操作后的停留(秒)
|
||||
},
|
||||
|
||||
# 轮间休息
|
||||
'round_break': {
|
||||
'min_break': 5, # 轮间最短休息(秒)
|
||||
'max_break': 20, # 轮间最长休息(秒)
|
||||
},
|
||||
|
||||
# ★ 反检测 — 拟人化操作策略(2026-05-04)
|
||||
'anti_detection': {
|
||||
'card_position_range': [0, 4], # 点卡片时选第几个(0=第1个)
|
||||
'search_result_range': [1, 4], # 搜索结果点第几个(0=第1个)
|
||||
'use_keyboard_like': False, # 是否用键盘 L 键点赞(预留)
|
||||
'use_close_button': False, # 是否用X按钮返回(预留)
|
||||
'scroll_method': 'mouse', # 切换方式: mouse/keyboard/js
|
||||
},
|
||||
|
||||
# 异常处理
|
||||
'retry': {
|
||||
'max_retries': 2, # 操作失败最大重试次数
|
||||
'retry_delay': [1, 3], # 重试前等待(秒)
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class BehaviorConfig:
|
||||
"""行为配置 — 控制所有操作的拟人化参数"""
|
||||
|
||||
def __init__(self, config: Optional[dict] = None):
|
||||
self._cfg = DEFAULT_CONFIG.copy()
|
||||
if config:
|
||||
self._deep_merge(self._cfg, config)
|
||||
|
||||
def _deep_merge(self, base: dict, override: dict):
|
||||
"""递归合并配置"""
|
||||
for key, val in override.items():
|
||||
if key in base and isinstance(base[key], dict) and isinstance(val, dict):
|
||||
self._deep_merge(base[key], val)
|
||||
else:
|
||||
base[key] = val
|
||||
|
||||
# ── 延迟方法 ──────────────────────────────────────────
|
||||
|
||||
def action_delay(self) -> float:
|
||||
"""两个操作之间的随机延迟"""
|
||||
base = self._cfg['base_delay']
|
||||
variance = self._cfg['delay_variance']
|
||||
return max(0.3, base + random.uniform(-variance, variance))
|
||||
|
||||
def sequence_gap(self) -> float:
|
||||
"""操作序列之间的间隔"""
|
||||
sg = self._cfg['sequence']
|
||||
return random.uniform(sg['min_gap'], sg['max_gap'])
|
||||
|
||||
def click_delay(self) -> float:
|
||||
"""点击前的微停顿"""
|
||||
cd = self._cfg['mouse']['click_delay']
|
||||
return random.uniform(cd[0], cd[1])
|
||||
|
||||
# ── 滚动方法 ──────────────────────────────────────────
|
||||
|
||||
def scroll_distance(self) -> int:
|
||||
"""每次滚动的距离"""
|
||||
dr = self._cfg['scroll']['distance_range']
|
||||
return random.randint(dr[0], dr[1])
|
||||
|
||||
def should_pause_scroll(self) -> bool:
|
||||
"""滚动中是否应该暂停"""
|
||||
return random.random() < self._cfg['scroll']['pause_chance']
|
||||
|
||||
def scroll_pause(self) -> float:
|
||||
"""滚动暂停时长"""
|
||||
pd = self._cfg['scroll']['pause_duration']
|
||||
return random.uniform(pd[0], pd[1])
|
||||
|
||||
# ── 注意力方法 ────────────────────────────────────────
|
||||
|
||||
def watch_duration(self) -> float:
|
||||
"""观看一个视频的时长"""
|
||||
wd = self._cfg['attention']['watch_duration']
|
||||
return random.uniform(wd[0], wd[1])
|
||||
|
||||
def should_distract(self) -> bool:
|
||||
"""是否应该"分心"(模拟走神)"""
|
||||
return random.random() < self._cfg['attention']['distraction_chance']
|
||||
|
||||
def distraction_duration(self) -> float:
|
||||
"""分心持续时长"""
|
||||
dd = self._cfg['attention']['distraction_duration']
|
||||
return random.uniform(dd[0], dd[1])
|
||||
|
||||
def linger_after_action(self) -> float:
|
||||
"""操作后的停留"""
|
||||
la = self._cfg['attention']['linger_after_action']
|
||||
|
||||
# ── 反检测方法 ────────────────────────────────────────
|
||||
|
||||
def card_position(self) -> int:
|
||||
"""随机选视频卡片位置(避免每次都点第1个)"""
|
||||
cr = self._cfg['anti_detection']['card_position_range']
|
||||
return random.randint(cr[0], cr[1])
|
||||
|
||||
def search_result_position(self) -> int:
|
||||
"""随机选搜索结果位置(避免每次都点第1个)"""
|
||||
sr = self._cfg['anti_detection']['search_result_range']
|
||||
return random.randint(sr[0], sr[1])
|
||||
return random.uniform(la[0], la[1])
|
||||
|
||||
# ── 轮间休息 ──────────────────────────────────────────
|
||||
|
||||
def round_break(self) -> float:
|
||||
"""轮间休息时长"""
|
||||
rb = self._cfg['round_break']
|
||||
return random.uniform(rb['min_break'], rb['max_break'])
|
||||
|
||||
# ── 操作顺序 ──────────────────────────────────────────
|
||||
|
||||
def should_shuffle(self) -> bool:
|
||||
"""是否打乱操作顺序"""
|
||||
return self._cfg['sequence']['shuffle']
|
||||
|
||||
def should_repeat(self) -> bool:
|
||||
"""是否重复上一个操作"""
|
||||
return random.random() < self._cfg['sequence']['repetition_chance']
|
||||
|
||||
# ── 重试 ──────────────────────────────────────────────
|
||||
|
||||
def retry_delay(self) -> float:
|
||||
"""重试前等待"""
|
||||
rd = self._cfg['retry']['retry_delay']
|
||||
return random.uniform(rd[0], rd[1])
|
||||
|
||||
@property
|
||||
def max_retries(self) -> int:
|
||||
return self._cfg['retry']['max_retries']
|
||||
|
||||
# ── 工具方法 ──────────────────────────────────────────
|
||||
|
||||
def sleep(self, seconds: float):
|
||||
"""带拟人化的 sleep(加入微抖动)"""
|
||||
jitter = random.uniform(-0.05, 0.05)
|
||||
time.sleep(max(0, seconds + jitter))
|
||||
@@ -0,0 +1,294 @@
|
||||
"""
|
||||
calibration.py — 校准触发器 + AI 兜底
|
||||
|
||||
功能:
|
||||
1. 定期检查锚点检测成功率
|
||||
2. 触发校准(连续失败/成功率低)
|
||||
3. AI 兜底分析(基于 DOM 数据)
|
||||
4. 生成校准候选规则
|
||||
"""
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from . import anchor_db as db
|
||||
|
||||
# oMLX 配置
|
||||
OLLAMA_BASE = "http://localhost:8000"
|
||||
OLLAMA_API_KEY = "5omlx"
|
||||
LLM_MODEL = "mlx-community--Qwen3.5-4B-MLX-4bit" # 或 gemma-3-4b-it-4bit
|
||||
|
||||
|
||||
# ── 校准检查 ──
|
||||
|
||||
def check_calibration(identity: str) -> dict:
|
||||
"""检查是否需要校准
|
||||
|
||||
Returns:
|
||||
dict: {needs, accuracy, consecutive_failures, level}
|
||||
"""
|
||||
accuracy = db.get_recent_accuracy(identity, count=50)
|
||||
consecutive_fail = db.get_recent_failures(identity, threshold=5)
|
||||
|
||||
# 判断级别
|
||||
if accuracy >= 0.9:
|
||||
level = "normal"
|
||||
elif accuracy >= 0.6:
|
||||
level = "warn"
|
||||
elif accuracy >= 0.3:
|
||||
level = "error"
|
||||
else:
|
||||
level = "alert"
|
||||
|
||||
needs = (accuracy < 0.6) or consecutive_fail
|
||||
|
||||
return {
|
||||
"needs_calibration": needs,
|
||||
"accuracy": round(accuracy, 2),
|
||||
"consecutive_failures": consecutive_fail,
|
||||
"level": level,
|
||||
}
|
||||
|
||||
|
||||
def should_trigger(identity: str) -> bool:
|
||||
"""快速判断是否应触发校准"""
|
||||
acc = db.get_recent_accuracy(identity, count=20)
|
||||
fail = db.get_recent_failures(identity, threshold=5)
|
||||
return acc < 0.6 or fail
|
||||
|
||||
|
||||
# ── AI 兜底 ──
|
||||
|
||||
def _build_ai_prompt(anchors: dict, screenshot_path: str = None) -> str:
|
||||
"""构建发给 LLM 的 prompt"""
|
||||
# 清理敏感数据
|
||||
safe_anchors = {k: v for k, v in anchors.items()
|
||||
if k not in ('body_text', 'video_src_prefix')}
|
||||
|
||||
prompt = f"""你是一个抖音页面状态分析专家。请根据以下页面DOM数据,判断当前页面处于什么状态。
|
||||
|
||||
DOM数据:
|
||||
```json
|
||||
{json.dumps(safe_anchors, ensure_ascii=False, indent=2)}
|
||||
```
|
||||
|
||||
请返回以下JSON格式的分析结果(不要包含其他内容):
|
||||
```json
|
||||
{{
|
||||
"state": "当前页面状态(HOME/PLAYER/SEARCH/SEARCH_PLAYER/DETAIL/POPUP/UNKNOWN)",
|
||||
"confidence": 0.95,
|
||||
"evidence": ["判断依据1", "判断依据2"],
|
||||
"suggested_anchors": [
|
||||
{{"field": "video_count", "operator": "gte", "value": "3", "reason": "为什么这个锚点有效"}}
|
||||
],
|
||||
"next_action": {{
|
||||
"op": "建议执行的操作(swipe_down/click_card/like/search/go_home/dismiss_popup)",
|
||||
"reason": "为什么建议这个操作"
|
||||
}}
|
||||
}}
|
||||
```
|
||||
|
||||
注意:
|
||||
- HOME: 首页/精选页,video_count=1,card_count>0
|
||||
- PLAYER: 视频播放页,video_count>=3,有视频在播放
|
||||
- SEARCH: 搜索结果页,title含"搜索"
|
||||
- SEARCH_PLAYER: 从搜索结果进入的播放页
|
||||
- 如果DOM数据异常(如video_count=0),可能是页面未加载或崩溃"""
|
||||
return prompt
|
||||
|
||||
|
||||
async def ai_analyze(anchors: dict, screenshot_path: str = None) -> dict:
|
||||
"""调用本地LLM分析当前页面状态
|
||||
|
||||
Args:
|
||||
anchors: DOM 锚点数据
|
||||
screenshot_path: 截图路径(暂时不用,模型不支持vision)
|
||||
|
||||
Returns:
|
||||
dict: {state, confidence, evidence, suggested_anchors, next_action}
|
||||
"""
|
||||
prompt = _build_ai_prompt(anchors, screenshot_path)
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.post(
|
||||
f"{OLLAMA_BASE}/v1/chat/completions",
|
||||
headers={
|
||||
"Authorization": f"Bearer {OLLAMA_API_KEY}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={
|
||||
"model": LLM_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": "你是一个抖音页面状态分析专家,只输出JSON。"},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"temperature": 0.1,
|
||||
"max_tokens": 1000,
|
||||
},
|
||||
)
|
||||
result = resp.json()
|
||||
content = result["choices"][0]["message"]["content"]
|
||||
|
||||
# 提取 JSON
|
||||
# 先尝试直接解析
|
||||
try:
|
||||
parsed = json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
# 尝试从 ```json ... ``` 中提取
|
||||
import re
|
||||
m = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', content, re.DOTALL)
|
||||
if m:
|
||||
parsed = json.loads(m.group(1))
|
||||
else:
|
||||
return {
|
||||
"state": "UNKNOWN",
|
||||
"confidence": 0,
|
||||
"error": f"LLM返回格式异常: {content[:200]}",
|
||||
}
|
||||
|
||||
return parsed
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"state": "UNKNOWN",
|
||||
"confidence": 0,
|
||||
"error": str(e),
|
||||
}
|
||||
|
||||
|
||||
# ── 校准流程 ──
|
||||
|
||||
async def run_calibration(identity: str, anchors: dict,
|
||||
screenshot_path: str = None) -> dict:
|
||||
"""执行完整校准流程
|
||||
|
||||
1. 检查当前成功率
|
||||
2. AI 分析当前页面
|
||||
3. 生成校准候选
|
||||
4. 返回校准结果
|
||||
"""
|
||||
check = check_calibration(identity)
|
||||
print(f"[校准] {identity} 准确率: {check['accuracy']}, 级别: {check['level']}")
|
||||
|
||||
# AI 分析
|
||||
print(f"[校准] AI分析中...")
|
||||
ai_result = await ai_analyze(anchors, screenshot_path)
|
||||
ai_state = ai_result.get("state", "UNKNOWN")
|
||||
ai_confidence = ai_result.get("confidence", 0)
|
||||
print(f"[校准] AI判断: {ai_state} (置信度: {ai_confidence})")
|
||||
|
||||
# 如果 AI 建议了新锚点,生成校准候选
|
||||
suggested = ai_result.get("suggested_anchors", [])
|
||||
if suggested and ai_confidence >= 0.7:
|
||||
# 获取当前规则作为 old_rules
|
||||
old_rules = db.get_rules(ai_state)
|
||||
|
||||
candidate_id = db.add_candidate(
|
||||
state_name=ai_state,
|
||||
source="ai",
|
||||
old_rules=[dict(r) for r in old_rules],
|
||||
new_rules=suggested,
|
||||
)
|
||||
print(f"[校准] 候选规则已生成 (id={candidate_id})")
|
||||
|
||||
return {
|
||||
"calibration_triggered": True,
|
||||
"candidate_id": candidate_id,
|
||||
"ai_state": ai_state,
|
||||
"ai_confidence": ai_confidence,
|
||||
"check": check,
|
||||
"ai_result": ai_result,
|
||||
}
|
||||
|
||||
return {
|
||||
"calibration_triggered": False,
|
||||
"check": check,
|
||||
"ai_result": ai_result,
|
||||
}
|
||||
|
||||
|
||||
# ── 候选规则审批 ──
|
||||
|
||||
async def test_candidate(candidate_id: int, test_data: list) -> dict:
|
||||
"""用测试数据验证候选规则
|
||||
|
||||
Args:
|
||||
candidate_id: 校准候选ID
|
||||
test_data: [{anchors: dict, expected_state: str}, ...]
|
||||
|
||||
Returns:
|
||||
{candidate_id, old_accuracy, new_accuracy, accepted}
|
||||
"""
|
||||
candidate_rows = db.get_conn().execute(
|
||||
"SELECT * FROM calibration_candidates WHERE id=?", (candidate_id,)
|
||||
).fetchall()
|
||||
if not candidate_rows:
|
||||
return {"error": "candidate not found"}
|
||||
candidate = dict(candidate_rows[0])
|
||||
new_rules = json.loads(candidate["new_rules"])
|
||||
old_rules = json.loads(candidate["old_rules"])
|
||||
|
||||
# 不需要真的测试,直接在调用方处理
|
||||
return {
|
||||
"candidate_id": candidate_id,
|
||||
"new_rules": new_rules,
|
||||
"old_rules": old_rules,
|
||||
}
|
||||
|
||||
|
||||
def accept_candidate(candidate_id: int):
|
||||
"""采纳候选规则"""
|
||||
candidate_rows = db.get_conn().execute(
|
||||
"SELECT * FROM calibration_candidates WHERE id=?", (candidate_id,)
|
||||
).fetchall()
|
||||
if not candidate_rows:
|
||||
return
|
||||
candidate = dict(candidate_rows[0])
|
||||
new_rules = json.loads(candidate["new_rules"])
|
||||
state_name = candidate["state_name"]
|
||||
|
||||
# 停用旧规则
|
||||
for r in db.get_rules(state_name):
|
||||
db.update_rule(r["id"], active=0)
|
||||
|
||||
# 添加新规则
|
||||
for rule in new_rules:
|
||||
db.add_rule(
|
||||
state_name=state_name,
|
||||
field=rule["field"],
|
||||
operator=rule["operator"],
|
||||
value=rule["value"],
|
||||
logic_group=rule.get("logic_group", 0),
|
||||
)
|
||||
|
||||
db.update_candidate(candidate_id, "accepted")
|
||||
print(f"[校准] 候选#{candidate_id} 已采纳,{state_name}规则已更新")
|
||||
|
||||
|
||||
def reject_candidate(candidate_id: int):
|
||||
"""拒绝候选规则"""
|
||||
db.update_candidate(candidate_id, "rejected")
|
||||
print(f"[校准] 候选#{candidate_id} 已拒绝")
|
||||
|
||||
|
||||
# ── 快速测试(不依赖外部LLM)──
|
||||
|
||||
async def quick_fallback(anchors: dict) -> dict:
|
||||
"""快速兜底:基于简单规则判断(不调LLM),用于LLM不可用时的降级"""
|
||||
vc = anchors.get("video_count", 0)
|
||||
title = anchors.get("title", "")
|
||||
url = anchors.get("url", "")
|
||||
|
||||
if vc >= 3 and ("搜索" in title or "/search/" in url):
|
||||
return {"state": "SEARCH_PLAYER", "confidence": 0.7, "reason": "video≥3且来自搜索"}
|
||||
if vc >= 3:
|
||||
return {"state": "PLAYER", "confidence": 0.8, "reason": "video≥3"}
|
||||
if "搜索" in title and "精选" not in title:
|
||||
return {"state": "SEARCH", "confidence": 0.8, "reason": "标题含搜索"}
|
||||
if "精选" in title:
|
||||
return {"state": "HOME", "confidence": 0.7, "reason": "标题含精选"}
|
||||
return {"state": "UNKNOWN", "confidence": 0.3, "reason": "无法判断"}
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
"""
|
||||
comment_corpus.py — 评论语料库引擎
|
||||
|
||||
从 comment_corpus.yaml 加载分类评论,按权重随机选取。
|
||||
支持模板填充、分类筛选、手动增加。
|
||||
"""
|
||||
import random
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
|
||||
CORPUS_PATH = Path(__file__).parent.parent.parent / "config" / "comment_corpus.yaml"
|
||||
|
||||
|
||||
def load_corpus() -> dict:
|
||||
"""加载语料库 YAML"""
|
||||
with open(CORPUS_PATH, encoding='utf-8') as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def get_comment(keyword: str = "", category: str = None) -> str:
|
||||
"""获取一条随机评论
|
||||
|
||||
Args:
|
||||
keyword: 当前视频关键词,用于模板替换
|
||||
category: 指定分类(None=从所有启用分类中选)
|
||||
|
||||
Returns:
|
||||
评论文本
|
||||
"""
|
||||
corpus = load_corpus()
|
||||
cfg = corpus.get("config", {})
|
||||
enabled = cfg.get("enabled_categories", [])
|
||||
disabled = cfg.get("disabled_categories", [])
|
||||
prefer_templates = cfg.get("prefer_templates", True)
|
||||
|
||||
# 筛选可用分类
|
||||
categories = []
|
||||
for cat_key, cat_data in corpus.get("categories", {}).items():
|
||||
weight = cat_data.get("weight", 0)
|
||||
if weight <= 0:
|
||||
continue
|
||||
if enabled and cat_key not in enabled:
|
||||
continue
|
||||
if cat_key in disabled:
|
||||
continue
|
||||
if category and cat_key != category:
|
||||
continue
|
||||
categories.append((cat_key, cat_data, weight))
|
||||
|
||||
if not categories:
|
||||
return "好内容" # 兜底
|
||||
|
||||
# 按权重选分类
|
||||
keys, datas, weights = zip(*categories)
|
||||
chosen_key = random.choices(keys, weights=weights, k=1)[0]
|
||||
chosen_data = datas[keys.index(chosen_key)]
|
||||
|
||||
# 从选中的分类中选一条评论
|
||||
comments = chosen_data.get("comments", [])
|
||||
templates = chosen_data.get("templates", [])
|
||||
|
||||
# 优先用模板
|
||||
if prefer_templates and templates and random.random() < 0.4:
|
||||
template = random.choice(templates)
|
||||
return template.format(keyword=keyword or "这个")
|
||||
elif comments:
|
||||
return random.choice(comments)
|
||||
elif templates:
|
||||
template = random.choice(templates)
|
||||
return template.format(keyword=keyword or "这个")
|
||||
return "👍"
|
||||
|
||||
|
||||
def add_comment(category: str, comment: str):
|
||||
"""手动增加一条评论到语料库"""
|
||||
with open(CORPUS_PATH, encoding='utf-8') as f:
|
||||
corpus = yaml.safe_load(f)
|
||||
|
||||
cat = corpus.get("categories", {}).get(category)
|
||||
if not cat:
|
||||
print(f"❌ 分类 '{category}' 不存在")
|
||||
return
|
||||
|
||||
if "comments" not in cat:
|
||||
cat["comments"] = []
|
||||
cat["comments"].append(comment)
|
||||
|
||||
with open(CORPUS_PATH, 'w', encoding='utf-8') as f:
|
||||
yaml.dump(corpus, f, allow_unicode=True, indent=2, sort_keys=False)
|
||||
print(f"✅ 已添加评论到 [{category}]: {comment}")
|
||||
|
||||
|
||||
def list_categories() -> str:
|
||||
"""列出所有可用分类"""
|
||||
corpus = load_corpus()
|
||||
lines = ["📋 语料库分类:"]
|
||||
for key, data in corpus.get("categories", {}).items():
|
||||
w = data.get("weight", 0)
|
||||
status = "✅" if w > 0 else "⛔"
|
||||
n = len(data.get("comments", [])) + len(data.get("templates", []))
|
||||
lines.append(f" {status} {key:20s} (权重{w}, {n}条) - {data.get('label','')}")
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(list_categories())
|
||||
print()
|
||||
for _ in range(5):
|
||||
print(f" → {get_comment('编程')}")
|
||||
Binary file not shown.
@@ -0,0 +1,109 @@
|
||||
-- anchor.db 数据库设计 v1.0
|
||||
-- 锚点规则 + 原子操作 + 检测日志 + 校准
|
||||
|
||||
-- ── 1. 锚点规则表 ──
|
||||
-- 每个状态由多条规则组成(同 state + logic_group = AND,不同 group = OR)
|
||||
CREATE TABLE anchor_rules (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
state_name TEXT NOT NULL, -- HOME / PLAYER / SEARCH / DETAIL / POPUP
|
||||
field TEXT NOT NULL, -- 检测字段: video_count, title, video_playing, card_count, url, has_search_input
|
||||
operator TEXT NOT NULL, -- eq / neq / gt / gte / lt / lte / contains / not_contains / regex
|
||||
value TEXT NOT NULL, -- 目标值(统一存字符串)
|
||||
logic_group INTEGER DEFAULT 0, -- 0=AND组, 同组内AND, 不同组OR
|
||||
active INTEGER DEFAULT 1, -- 1=启用, 0=停用
|
||||
created_at TEXT DEFAULT (datetime('now','localtime')),
|
||||
updated_at TEXT DEFAULT (datetime('now','localtime'))
|
||||
);
|
||||
|
||||
-- ── 2. 原子操作表 ──
|
||||
-- 每个原子操作 = 一个操作标识 + 执行方式 + 验证规则
|
||||
CREATE TABLE atomic_ops (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
op_name TEXT NOT NULL UNIQUE, -- click_card, swipe_down, like, search, go_home, dismiss_popup
|
||||
description TEXT, -- 操作描述
|
||||
action_type TEXT DEFAULT 'locator', -- locator / keyboard / evaluate / mouse
|
||||
target TEXT, -- CSS选择器 / 按键名 / JS代码
|
||||
verify_rule TEXT, -- 验证方式: video_src_change / video_count_gt / title_change
|
||||
retry_count INTEGER DEFAULT 3,
|
||||
fallback_op TEXT, -- 失败时降级的操作名
|
||||
created_at TEXT DEFAULT (datetime('now','localtime')),
|
||||
updated_at TEXT DEFAULT (datetime('now','localtime'))
|
||||
);
|
||||
|
||||
-- ── 3. 状态迁移表 ──
|
||||
-- 定义合法的状态转换(状态机)
|
||||
CREATE TABLE state_transitions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
from_state TEXT NOT NULL, -- HOME, PLAYER, SEARCH
|
||||
to_state TEXT NOT NULL, -- PLAYER, SEARCH, HOME
|
||||
via_op TEXT NOT NULL, -- click_card, swipe_down, search, go_home
|
||||
UNIQUE(from_state, to_state, via_op)
|
||||
);
|
||||
|
||||
-- ── 4. 检测日志表 ──
|
||||
-- 每次锚点检测的记录,用于成功率追踪
|
||||
CREATE TABLE detection_logs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
identity TEXT, -- douyin_01, douyin_02
|
||||
expected_state TEXT, -- 操作前预期的状态
|
||||
detected_state TEXT, -- 实际检测到的状态
|
||||
match INTEGER, -- 1=匹配, 0=不匹配
|
||||
anchor_snapshot TEXT, -- 锚点数据快照(JSON)
|
||||
screenshot_path TEXT, -- 截图路径
|
||||
timestamp TEXT DEFAULT (datetime('now','localtime'))
|
||||
);
|
||||
CREATE INDEX idx_detection_logs_match ON detection_logs(match, timestamp);
|
||||
|
||||
-- ── 5. 校准候选表 ──
|
||||
-- AI或自动校准产生的新规则,对比验证后决定是否采纳
|
||||
CREATE TABLE calibration_candidates (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
state_name TEXT, -- 哪个状态的规则
|
||||
source TEXT, -- ai / manual / auto_recovery
|
||||
old_rules TEXT, -- 旧规则JSON
|
||||
new_rules TEXT, -- 建议的新规则JSON
|
||||
test_count INTEGER DEFAULT 0, -- 测试次数
|
||||
test_pass INTEGER DEFAULT 0, -- 测试通过次数
|
||||
status TEXT DEFAULT 'pending', -- pending / accepted / rejected / testing
|
||||
created_at TEXT DEFAULT (datetime('now','localtime'))
|
||||
);
|
||||
|
||||
-- ── 初始数据:基于2026-05-06手动采集的锚点 ──
|
||||
|
||||
-- HOME 状态规则
|
||||
INSERT INTO anchor_rules (state_name, field, operator, value, logic_group) VALUES
|
||||
('HOME', 'video_count', 'eq', '1', 0),
|
||||
('HOME', 'title', 'contains', '抖音精选', 0);
|
||||
|
||||
-- PLAYER 状态规则(全屏播放)
|
||||
INSERT INTO anchor_rules (state_name, field, operator, value, logic_group) VALUES
|
||||
('PLAYER', 'video_count', 'gte', '3', 0),
|
||||
('PLAYER', 'video_playing', 'eq', 'true', 0);
|
||||
|
||||
-- SEARCH 状态规则
|
||||
INSERT INTO anchor_rules (state_name, field, operator, value, logic_group) VALUES
|
||||
('SEARCH', 'title', 'contains', '搜索', 0),
|
||||
('SEARCH', 'title', 'not_contains', '精选', 0);
|
||||
|
||||
-- SEARCH_PLAYER(从搜索结果进入的播放页)
|
||||
INSERT INTO anchor_rules (state_name, field, operator, value, logic_group) VALUES
|
||||
('SEARCH_PLAYER', 'video_count', 'gte', '3', 0),
|
||||
('SEARCH_PLAYER', 'video_playing', 'eq', 'true', 0),
|
||||
('SEARCH_PLAYER', 'url', 'contains', '/search/', 0);
|
||||
|
||||
-- 原子操作
|
||||
INSERT INTO atomic_ops (op_name, description, action_type, target, verify_rule, retry_count) VALUES
|
||||
('click_card', '点击首页视频卡片', 'locator', '.discover-video-card-item', 'video_count_gte_3', 3),
|
||||
('swipe_down', '下滑切换视频', 'keyboard', 'ArrowDown', 'video_src_change', 2),
|
||||
('like', '点赞当前视频', 'locator', '[data-e2e="like-count"]', 'like_count_change', 1),
|
||||
('search', '搜索关键词', 'evaluate', 'search_input', 'title_contains_搜索', 2),
|
||||
('go_home', '回到首页', 'locator', 'https://www.douyin.com/', 'video_count_eq_1', 2),
|
||||
('dismiss_popup','关闭弹窗', 'locator', 'button:text-is("我知道了")','popup_gone', 2);
|
||||
|
||||
-- 状态迁移
|
||||
INSERT INTO state_transitions (from_state, to_state, via_op) VALUES
|
||||
('HOME', 'PLAYER', 'click_card'),
|
||||
('PLAYER', 'PLAYER', 'swipe_down'),
|
||||
('PLAYER', 'HOME', 'go_home'),
|
||||
('HOME', 'SEARCH', 'search'),
|
||||
('SEARCH', 'SEARCH_PLAYER', 'click_card');
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,152 @@
|
||||
"""
|
||||
ui_layout.py — 抖音评论区 UI 布局参数 (v1.0.0)
|
||||
|
||||
设计原则(2026-05-09 基于真人操作反馈):
|
||||
- 不硬编码绝对坐标
|
||||
- 所有坐标相对于浏览器窗口右边缘、底边缘计算
|
||||
- 布局参数可配置(支持 per-account 覆盖)
|
||||
- 分为:DOM检测优先 → 坐标落位保底 → 状态验证确认
|
||||
|
||||
抖音评论区布局(KeyX 打开后):
|
||||
┌──────────────────────────────────────┐
|
||||
│ 评论面板 (固定宽度 ~500-600px) │
|
||||
│ │
|
||||
│ 评论列表区域 │
|
||||
│ │
|
||||
│ ┌──────────────────┬──┬──┬──────┐ │
|
||||
│ │ 输入框区域 │😊│@│ 发送 │ │
|
||||
│ │ 留下你的精彩评论吧 │ │ │ │ │
|
||||
│ └──────────────────┴──┴──┴──────┘ │
|
||||
└──────────────────────────────────────┘
|
||||
|
||||
关键:
|
||||
- 输入框在评论面板底部靠左
|
||||
- Emoji/ Mention/ Send 按钮在右侧
|
||||
- 输入框宽度 = 评论面板宽度 - 右侧按钮区(~130px)
|
||||
- 点击位置应落在输入框中间偏左区域
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional, Tuple
|
||||
|
||||
__version__ = "1.0.0"
|
||||
|
||||
|
||||
# ── 默认布局参数(基于 702×783 窗口测量) ─────────────────────
|
||||
|
||||
DEFAULT_LAYOUT = {
|
||||
# 评论面板布局(相对于窗口左边缘)
|
||||
"comment_panel": {
|
||||
"width_ratio": 0.85, # 面板占窗口宽度的比例
|
||||
"left_offset": 0, # 面板距离左边缘(像素固定值)
|
||||
},
|
||||
|
||||
# 输入框位置(相对于窗口右边缘,因为按钮在右侧)
|
||||
"input_box": {
|
||||
# 从右边缘往左偏移量(校准值 2026-05-10)
|
||||
"from_right_margin": 223, # 702-479=223(实测 activeElement 中心)
|
||||
# 从上边缘往下偏移量
|
||||
"from_top_offset": "auto", # "auto" = 动态检测,fallback 使用下面固定值
|
||||
"from_top_fallback": 687, # 实测 activeElement Y 中心
|
||||
"height": 36, # 输入框高度
|
||||
},
|
||||
|
||||
# 按钮区域(从右边缘往左)
|
||||
"buttons": {
|
||||
"send_width": 60, # 发送按钮宽度
|
||||
"icon_width": 30, # emoji/@ 图标宽度
|
||||
"button_count": 3, # 按钮数量 (emoji + @ + 发送)
|
||||
},
|
||||
|
||||
# 发送键
|
||||
"send_key": {
|
||||
"method": "dom", # dom/alt_enter。DOM 优先找"发送"按钮
|
||||
"alt_enter_fallback": True,
|
||||
},
|
||||
|
||||
# 验证码弹窗
|
||||
"verify_code": {
|
||||
"input_placeholder": "验证码",
|
||||
"confirm_button_texts": ["确认", "提交", "验证"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_layout(account_config: Optional[Dict] = None) -> dict:
|
||||
"""获取 UI 布局参数(支持 per-account 覆盖)
|
||||
|
||||
Args:
|
||||
account_config: accounts.yaml 中的账号配置,可包含 ui_layout 覆盖
|
||||
|
||||
Returns:
|
||||
合并后的布局参数字典
|
||||
"""
|
||||
layout = dict(DEFAULT_LAYOUT)
|
||||
if account_config and "ui_layout" in account_config:
|
||||
# 深度合并
|
||||
_deep_merge(layout, account_config["ui_layout"])
|
||||
return layout
|
||||
|
||||
|
||||
def _deep_merge(base: dict, override: dict):
|
||||
"""递归合并字典(不覆盖子键未指定的部分)"""
|
||||
for key, val in override.items():
|
||||
if key in base and isinstance(base[key], dict) and isinstance(val, dict):
|
||||
_deep_merge(base[key], val)
|
||||
else:
|
||||
base[key] = val
|
||||
|
||||
|
||||
def calc_input_position(window_width: int, window_height: int,
|
||||
layout: Optional[Dict] = None) -> Tuple[int, int]:
|
||||
"""计算评论输入框的目标点击坐标
|
||||
|
||||
Args:
|
||||
window_width: 浏览器窗口宽度 (如 702)
|
||||
window_height: 浏览器窗口高度 (如 783)
|
||||
|
||||
Returns:
|
||||
(x, y) — 基于 viewport 的绝对坐标
|
||||
"""
|
||||
if layout is None:
|
||||
layout = DEFAULT_LAYOUT
|
||||
|
||||
ib = layout["input_box"]
|
||||
|
||||
# X坐标:从窗口右边缘往左偏移
|
||||
x = window_width - ib["from_right_margin"]
|
||||
|
||||
# Y坐标
|
||||
y = ib.get("from_top_fallback", 720)
|
||||
|
||||
return (x, y)
|
||||
|
||||
|
||||
def calc_panel_rect(window_width: int, window_height: int,
|
||||
layout: Optional[Dict] = None) -> Tuple[int, int, int, int]:
|
||||
"""计算评论面板的包围矩形(用于 debug / 视觉确认)
|
||||
|
||||
Returns:
|
||||
(left, top, width, height)
|
||||
"""
|
||||
if layout is None:
|
||||
layout = DEFAULT_LAYOUT
|
||||
panel = layout["comment_panel"]
|
||||
pw = int(window_width * panel["width_ratio"])
|
||||
return (panel["left_offset"], 0, pw, window_height)
|
||||
|
||||
|
||||
def format_layout_summary(layout: Optional[Dict] = None) -> str:
|
||||
"""生成布局参数的摘要文本(用于日志)"""
|
||||
if layout is None:
|
||||
layout = DEFAULT_LAYOUT
|
||||
ib = layout["input_box"]
|
||||
btn = layout["buttons"]
|
||||
return (
|
||||
f"UI布局: "
|
||||
f"输入框右边缘偏移={ib['from_right_margin']}px, "
|
||||
f"Y坐标兜底={ib['from_top_fallback']}px, "
|
||||
f"按钮数={btn['button_count']}, "
|
||||
f"发送方式={layout['send_key']['method']}"
|
||||
)
|
||||
Reference in New Issue
Block a user