feat(monitor): 抖音接入博主监控
上游爬虫本身不缺抖音能力(三模式、四项指标、二级评论都与小红书对等、指标还是同名同列),
缺的全在监控层的适配。这次把「平台之间不一样」的管子集中到一个新模块,再把散落的
xhs 硬编码接上去。
* 新增 api/monitor/adapters.py:产物目录名、jsonl 字段别名、目标链接形态与正则、
通知链接模板。不放进 platforms.py 是因为那个模块被 describe_all() 整个序列化进
/api/config/platforms 交给前端,塞进正则和目录名会让爬虫内部细节漏进 API 载荷。
代价是两个注册表可能漂移,用一条测试钉住「声明接通就必须有适配器」。
* 两个必须知道的坑,都在这版里处理掉了:
1) 抖音的平台 id 是 dy,而 store 把产物写在 douyin/ 下(store/douyin/_store_impl.py:47)。
不改就是 ingest 一个文件都读不到 —— 不报错,只是 0 条,然后被冒充成「疑似登录失效」。
2) 抖音的作品没有 note_id(叫 aweme_id)、评论也用 aweme_id 指作品。ingest 第一步是
`if not note_id: continue`,不映射就逐条全丢。
另外抖音顶层评论的 parent_comment_id 是字符串 "0",归一成空串,免得前端多出悬空的父节点。
* 顺带把「东西抓到了、只是没落在期望目录里」单独识别出来。这类故障的现象和登录失效
一模一样,按登录失效报会把人指去查完全错误的方向。
* 修两个既有 bug(今天只有小红书所以无害,加抖音就踩响):
- service.py update_task 换目标时漏传 task.platform,回落到默认小红书
- scheduler.py 取 cookie 没传 platform,抖音任务会读着小红书那份 cookie 不动
* 行为变更(已与用户确认):cookie 闸门改成「没 cookie 且没开 CDP」才跳过。
CDP 模式下登录态来自被接管的浏览器,粘不粘 cookie 由不得它决定;不放行的话,
选了「接管已有 Chrome」却没粘 cookie 的用户会看到任务永远不触发,而且不报错。
副作用是开启了 CDP 的小红书任务也不再被该闸门拦住 —— 语义上是对的。
* 目标输入框的示例链接与措辞改由能力矩阵提供(notes_label 抖音说「作品」、小红书说
「笔记」;「建议只填纯 ID」是小红书专属劝告,抖音链接不带令牌,不再显示)。
测试 +22 条(858 通过),其中最关键的是「抖音作品/评论不被静默丢弃」与「产物目录名
不等于平台 id」两条 —— 都是把最难查的失败模式钉死在回归网里。
注意:抖音这条路的**端到端尚未验证**,需要一份可用的抖音登录态(CDP 那台 Chrome 里
登录,或导出一份 cookie)。单测覆盖的是解析与入库,真实抓取还没跑过。
This commit is contained in:
@@ -35,6 +35,7 @@ from sqlalchemy.pool import StaticPool
|
||||
|
||||
from tools.time_util import get_current_timestamp
|
||||
|
||||
from api.monitor import adapters
|
||||
from api.monitor.ingest import describe_exit_code, ingest_run, parse_count
|
||||
from api.monitor.models import (
|
||||
EVENT_AUTH_FAILURE,
|
||||
@@ -46,6 +47,7 @@ from api.monitor.models import (
|
||||
EVENT_RUN_FAILED,
|
||||
MODE_CREATOR,
|
||||
MonitorBase,
|
||||
MonitorComment,
|
||||
MonitorEvent,
|
||||
MonitorNote,
|
||||
MonitorNoteMetric,
|
||||
@@ -118,9 +120,14 @@ def _write_run_dir(
|
||||
root: Path,
|
||||
notes: List[Dict[str, Any]],
|
||||
comments: Optional[List[Dict[str, Any]]] = None,
|
||||
subdir: str = "xhs",
|
||||
) -> Path:
|
||||
"""Write a run's jsonl output in the crawler's own layout."""
|
||||
jsonl_dir = root / "xhs" / "jsonl"
|
||||
"""Write a run's jsonl output in the crawler's own layout.
|
||||
|
||||
``subdir`` 是**爬虫**落盘的目录名,不是监控层的平台 id —— 抖音那边这两者不同
|
||||
(平台 id 是 ``dy``、目录是 ``douyin``),所以必须能分开指定,否则测不出那个差异。
|
||||
"""
|
||||
jsonl_dir = root / subdir / "jsonl"
|
||||
jsonl_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
contents = jsonl_dir / "creator_contents_2026-01-01.jsonl"
|
||||
@@ -170,6 +177,50 @@ def _comment(comment_id: str, note_id: str, create_time: int, **extra) -> Dict[s
|
||||
return record
|
||||
|
||||
|
||||
def _dy_note(aweme_id: str, liked: Any = "10", **extra) -> Dict[str, Any]:
|
||||
"""抖音作品记录 —— 键名照抄 store/douyin/__init__.py 的落盘字段。
|
||||
|
||||
重点在于**没有** ``note_id``:抖音叫 ``aweme_id``。这一条差异没映射好,就是
|
||||
每条记录都被 ingest 悄悄 continue 掉、一条不剩。
|
||||
"""
|
||||
record = {
|
||||
"aweme_id": aweme_id,
|
||||
"aweme_type": "0",
|
||||
"title": f"title-{aweme_id}",
|
||||
"desc": f"title-{aweme_id}",
|
||||
"create_time": 1700000000000,
|
||||
"creator_hash": "hash",
|
||||
"nickname": "u***r",
|
||||
"liked_count": liked,
|
||||
"collected_count": "1",
|
||||
"comment_count": "1",
|
||||
"share_count": "1",
|
||||
"aweme_url": f"https://www.douyin.com/video/{aweme_id}",
|
||||
"cover_url": "https://img/cover.jpg",
|
||||
}
|
||||
record.update(extra)
|
||||
return record
|
||||
|
||||
|
||||
def _dy_comment(
|
||||
comment_id: str, aweme_id: str, create_time: int, **extra
|
||||
) -> Dict[str, Any]:
|
||||
record = {
|
||||
"comment_id": comment_id,
|
||||
"create_time": create_time,
|
||||
"aweme_id": aweme_id,
|
||||
"content": f"content-{comment_id}",
|
||||
"creator_hash": "hash",
|
||||
"nickname": "u***r",
|
||||
"sub_comment_count": "0",
|
||||
"like_count": "0",
|
||||
# 抖音顶层评论的父 id 是字符串 "0",不是空串。
|
||||
"parent_comment_id": "0",
|
||||
}
|
||||
record.update(extra)
|
||||
return record
|
||||
|
||||
|
||||
async def _events(db: AsyncSession, event_type: Optional[str] = None) -> List[MonitorEvent]:
|
||||
stmt = select(MonitorEvent)
|
||||
if event_type:
|
||||
@@ -525,3 +576,125 @@ class TestIdempotency:
|
||||
assert result.new_notes == 0
|
||||
assert result.new_comments == 0
|
||||
assert len(list((await db.scalars(select(MonitorNote))).all())) == notes_after_first
|
||||
|
||||
|
||||
class TestDouyinIngest:
|
||||
"""抖音的产物形状与小红书不同 —— 这里钉住「不会被静默丢掉」。
|
||||
|
||||
这一组存在的理由,是这个改动最危险的失败模式:字段名或目录名没对上时,ingest
|
||||
不报错,只是**一条都不入库**,然后被当成「疑似登录失效」报出去。
|
||||
"""
|
||||
|
||||
async def _ingest(
|
||||
self,
|
||||
db,
|
||||
tmp_path,
|
||||
notes,
|
||||
comments=None,
|
||||
platform="dy",
|
||||
subdir="douyin",
|
||||
):
|
||||
task = await _make_task(db, platform=platform)
|
||||
run = await _make_run(db, task, started_at=1)
|
||||
_write_run_dir(tmp_path, notes, comments, subdir=subdir)
|
||||
result = await ingest_run(db, run, task, tmp_path)
|
||||
return task, run, result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_notes_are_ingested_under_their_douyin_field_names(self, db, tmp_path):
|
||||
aweme_id = "7525082444551310602"
|
||||
_task, _run, result = await self._ingest(db, tmp_path, [_dy_note(aweme_id)])
|
||||
|
||||
note = await db.scalar(select(MonitorNote))
|
||||
assert note is not None, "抖音作品被静默丢弃了 —— 多半是 aweme_id 没映射到 note_id"
|
||||
assert note.note_id == aweme_id
|
||||
assert note.note_url == f"https://www.douyin.com/video/{aweme_id}"
|
||||
assert note.cover == "https://img/cover.jpg"
|
||||
assert note.source_kind == "0"
|
||||
assert note.published_at == 1700000000000
|
||||
assert result.notes_fetched == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_artifact_directory_is_not_the_platform_id(self, db, tmp_path):
|
||||
"""目录名与平台 id 不一致,是这套适配里最反直觉的一条。
|
||||
|
||||
抖音的平台 id 是 ``dy``,而爬虫把产物写在 ``douyin/`` 下。把它钉在这里,
|
||||
是为了让「顺手改成一致」这件事会在测试里红掉,而不是让 ingest 悄悄读 0 条。
|
||||
"""
|
||||
assert adapters.artifact_dir("dy") == "douyin"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_writing_into_the_platform_id_directory_reads_nothing(self, db, tmp_path):
|
||||
"""反面:产物落在 ``dy/`` 下时一条都读不到 —— 这正是映射要解决的问题。"""
|
||||
_task, _run, result = await self._ingest(db, tmp_path, [_dy_note("1")], subdir="dy")
|
||||
|
||||
assert result.notes_fetched == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_misplaced_output_is_blamed_on_the_directory_not_the_login(
|
||||
self, db, tmp_path
|
||||
):
|
||||
"""产物其实抓到了,只是目录名不对 —— 不该报成「疑似登录失效」。
|
||||
|
||||
这是最难查的一类故障:登录是好的、数据也抓到了,但报出来的现象和登录失效
|
||||
一模一样,会把人指去查完全错误的方向。
|
||||
"""
|
||||
_task, run, _result = await self._ingest(
|
||||
db, tmp_path, [_dy_note("1")], subdir="dy"
|
||||
)
|
||||
|
||||
assert any("目录" in event.title for event in await _events(db, EVENT_NO_DATA))
|
||||
assert await _events(db, EVENT_AUTH_FAILURE) == []
|
||||
assert run.error_message and "dy" in run.error_message
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_comments_are_linked_through_aweme_id(self, db, tmp_path):
|
||||
aweme_id = "7525082444551310602"
|
||||
_task, _run, result = await self._ingest(
|
||||
db,
|
||||
tmp_path,
|
||||
[_dy_note(aweme_id)],
|
||||
comments=[_dy_comment("c1", aweme_id, 500)],
|
||||
)
|
||||
|
||||
comment = await db.scalar(select(MonitorComment))
|
||||
assert comment is not None, "抖音评论被静默丢弃了 —— 多半是 aweme_id 没映射"
|
||||
assert comment.note_id == aweme_id
|
||||
assert result.comments_fetched == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_top_level_parent_of_zero_becomes_empty(self, db, tmp_path):
|
||||
"""抖音顶层评论的父 id 是 "0";原样存进去,前端会多出一堆悬空的父节点。"""
|
||||
aweme_id = "7525082444551310602"
|
||||
await self._ingest(
|
||||
db,
|
||||
tmp_path,
|
||||
[_dy_note(aweme_id)],
|
||||
comments=[
|
||||
_dy_comment("c1", aweme_id, 500),
|
||||
_dy_comment("c2", aweme_id, 600, parent_comment_id="c1"),
|
||||
],
|
||||
)
|
||||
|
||||
by_id = {c.comment_id: c for c in (await db.scalars(select(MonitorComment))).all()}
|
||||
assert by_id["c1"].parent_comment_id == ""
|
||||
assert by_id["c2"].parent_comment_id == "c1"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_four_metrics_need_no_mapping(self, db, tmp_path):
|
||||
"""四个指标键两边同名 —— 抖音作品照样进 monitor_note_metric,差分照常。"""
|
||||
aweme_id = "7525082444551310602"
|
||||
task = await _make_task(db, platform="dy")
|
||||
|
||||
_write_run_dir(tmp_path, [_dy_note(aweme_id, liked="100")], subdir="douyin")
|
||||
run1 = await _make_run(db, task, started_at=1)
|
||||
await ingest_run(db, run1, task, tmp_path)
|
||||
|
||||
metric = await db.scalar(select(MonitorNoteMetric))
|
||||
assert metric is not None and metric.liked_count == 100
|
||||
|
||||
_write_run_dir(tmp_path, [_dy_note(aweme_id, liked="150")], subdir="douyin")
|
||||
run2 = await _make_run(db, task, started_at=2)
|
||||
await ingest_run(db, run2, task, tmp_path)
|
||||
|
||||
assert len(await _events(db, EVENT_METRIC_DELTA)) == 1
|
||||
|
||||
Reference in New Issue
Block a user