feat(monitor): 博主的粉丝数,以及给作品起备注
两件都是「作品栏里把这东西认出来」的延伸: * **账号级指标**:作品列表只会说「这条涨了多少赞」,说不了「这个人整个 账号的粉丝在涨还是在掉」。抖音的资料接口本来就有粉丝数/总获赞/作品数, 每轮顺手记一条快照(`monitor_creator_stat`,粒度 = 任务×博主×轮次, 和作品指标同形)。组头显示最近一条。 快照在「一条作品都没采到」的早退**之前**落:作品列表被风控挡住的那一轮, 正是「粉丝还在涨、但新作品没在发现」最该被看见的时刻。 * **作品备注**:博主备注回答「这个账号是谁」,这条回答「这条我要盯着」。 一个博主底下常常只有一两件值得盯的作品,所以不能合并成一条。键取 (platform, note_id),跨任务共用一份。 两边都守住同一条口径:**不知道就是 null,不写成 0** —— 0 在趋势图上是一条 砸到底的线,和「还没采到」是两回事。
This commit is contained in:
@@ -54,6 +54,7 @@ from api.monitor.models import (
|
||||
MODE_CREATOR,
|
||||
MonitorBase,
|
||||
MonitorComment,
|
||||
MonitorCreatorStat,
|
||||
MonitorEvent,
|
||||
MonitorNote,
|
||||
MonitorNoteMetric,
|
||||
@@ -127,11 +128,15 @@ def _write_run_dir(
|
||||
notes: List[Dict[str, Any]],
|
||||
comments: Optional[List[Dict[str, Any]]] = None,
|
||||
subdir: str = "xhs",
|
||||
profiles: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> Path:
|
||||
"""Write a run's jsonl output in the crawler's own layout.
|
||||
|
||||
``subdir`` 是**爬虫**落盘的目录名,不是监控层的平台 id —— 抖音那边这两者不同
|
||||
(平台 id 是 ``dy``、目录是 ``douyin``),所以必须能分开指定,否则测不出那个差异。
|
||||
|
||||
``profiles`` 为 None 时**不写**这个文件(小红书那条路根本不产生它),给列表时写
|
||||
——包括空列表,那是「问了但没问到」。
|
||||
"""
|
||||
jsonl_dir = root / subdir / "jsonl"
|
||||
jsonl_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -147,6 +152,12 @@ def _write_run_dir(
|
||||
"\n".join(json.dumps(c, ensure_ascii=False) for c in comments),
|
||||
encoding="utf-8",
|
||||
)
|
||||
if profiles is not None:
|
||||
profile_file = jsonl_dir / "creator_profile_2026-01-01.jsonl"
|
||||
profile_file.write_text(
|
||||
"\n".join(json.dumps(p, ensure_ascii=False) for p in profiles),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return root
|
||||
|
||||
|
||||
@@ -829,3 +840,139 @@ class TestFailureDiagnosis:
|
||||
|
||||
events = await _events(db, EVENT_RUN_FAILED)
|
||||
assert "account blocked" in events[0].title
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 博主账号级指标(粉丝 / 总获赞 / 作品数)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _profile(creator_hash: str = "hash", **extra) -> Dict[str, Any]:
|
||||
"""``creator_profile_*.jsonl`` 里的一行 —— 形状由 douyin_api.author_profile 决定。"""
|
||||
record: Dict[str, Any] = {
|
||||
"creator_hash": creator_hash,
|
||||
"nickname": "博主",
|
||||
"unique_id": "abc",
|
||||
"fans": 12000,
|
||||
"total_favorited": 83000,
|
||||
"works": 42,
|
||||
"following": 7,
|
||||
}
|
||||
record.update(extra)
|
||||
return record
|
||||
|
||||
|
||||
class TestCreatorStatSnapshots:
|
||||
"""**账号级**指标和作品级指标是两回事:后者说"这条视频涨了多少赞",前者说
|
||||
"这个人整个账号的粉丝在涨还是在掉"。作品列表给不了后者,所以单独存一张表。
|
||||
"""
|
||||
|
||||
async def _ingest(self, db, tmp_path, notes, profiles, platform="dy", subdir="douyin"):
|
||||
task = await _make_task(db, platform=platform)
|
||||
run = await _make_run(db, task, started_at=1)
|
||||
_write_run_dir(tmp_path, notes, comments=[], subdir=subdir, profiles=profiles)
|
||||
result = await ingest_run(db, run, task, tmp_path)
|
||||
return task, run, result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_profile_becomes_a_snapshot(self, db, tmp_path):
|
||||
task, run, _result = await self._ingest(db, tmp_path, [_dy_note("1")], [_profile()])
|
||||
|
||||
stat = await db.scalar(select(MonitorCreatorStat))
|
||||
|
||||
assert stat is not None
|
||||
assert (stat.task_id, stat.run_id) == (task.id, run.id)
|
||||
assert stat.creator_hash == "hash"
|
||||
assert stat.nickname == "博主"
|
||||
assert (stat.fans, stat.total_favorited, stat.works_count) == (12000, 83000, 42)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_missing_count_stays_null_not_zero(self, db, tmp_path):
|
||||
"""0 是真实值(掉到零),null 是不知道。混起来趋势图就是在撒谎。"""
|
||||
await self._ingest(db, tmp_path, [_dy_note("1")], [_profile(fans=None)])
|
||||
|
||||
stat = await db.scalar(select(MonitorCreatorStat))
|
||||
|
||||
assert stat.fans is None
|
||||
# 同一个博主其它字段照常。
|
||||
assert stat.works_count == 42
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_abbreviated_counts_are_parsed(self, db, tmp_path):
|
||||
"""走的是和作品指标同一个 parse_count —— 平台给你「1.2万」也得认。"""
|
||||
await self._ingest(db, tmp_path, [_dy_note("1")], [_profile(fans="1.2万")])
|
||||
|
||||
assert (await db.scalar(select(MonitorCreatorStat))).fans == 12000
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_same_creator_twice_in_one_run_yields_one_snapshot(self, db, tmp_path):
|
||||
"""一个任务可以配多个目标,退化路径下它们可能落在同一个博主身上。
|
||||
|
||||
唯一键是 (task_id, creator_hash, run_id) —— 重复插入会撞键,把整轮炸掉。
|
||||
(和作品重复那次是同一类事故。)
|
||||
"""
|
||||
await self._ingest(
|
||||
db, tmp_path, [_dy_note("1")], [_profile(), _profile(nickname="另一条")]
|
||||
)
|
||||
|
||||
stats = list((await db.scalars(select(MonitorCreatorStat))).all())
|
||||
|
||||
assert len(stats) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_profile_without_a_hash_is_skipped(self, db, tmp_path):
|
||||
"""哈希都算不出来,这条快照谁也查不到,落下去只是垃圾。"""
|
||||
await self._ingest(db, tmp_path, [_dy_note("1")], [_profile(creator_hash="")])
|
||||
|
||||
assert (await db.scalars(select(MonitorCreatorStat))).all() == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_profile_file_is_fine(self, db, tmp_path):
|
||||
"""小红书那条路(爬虫进程)根本不产生这个文件 —— 不能因此报错。"""
|
||||
task = await _make_task(db) # xhs
|
||||
run = await _make_run(db, task, started_at=1)
|
||||
_write_run_dir(tmp_path, [_note("n1")], comments=[]) # 没有 profiles 参数
|
||||
|
||||
result = await ingest_run(db, run, task, tmp_path)
|
||||
|
||||
assert result.status == RUN_SUCCESS
|
||||
assert (await db.scalars(select(MonitorCreatorStat))).all() == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_stats_are_kept_even_when_no_works_were_fetched(self, db, tmp_path):
|
||||
"""**这条是这里最值得留的一个。**
|
||||
|
||||
作品列表被风控挡住时,这一轮一条作品都拿不到、run 会被判成失败。但博主的粉丝数
|
||||
并不因为这件事就不存在 —— 「粉丝还在涨,但新作品没在发现」恰恰是最该看见的时刻。
|
||||
快照要是挂在「作品采到了」后面,就正好在最需要它的那一轮丢掉。
|
||||
"""
|
||||
_task, run, result = await self._ingest(db, tmp_path, [], [_profile()])
|
||||
|
||||
assert result.status == RUN_PARTIAL # 一条作品都没有,这轮确实不算成功
|
||||
assert run.status == RUN_PARTIAL
|
||||
stat = await db.scalar(select(MonitorCreatorStat))
|
||||
assert stat is not None and stat.fans == 12000
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_each_run_adds_its_own_snapshot(self, db, tmp_path):
|
||||
"""趋势靠的就是这个:一条一轮,不要覆盖。"""
|
||||
task = await _make_task(db, platform="dy")
|
||||
first = await _make_run(db, task, started_at=1)
|
||||
_write_run_dir(tmp_path, [_dy_note("1")], comments=[], subdir="douyin",
|
||||
profiles=[_profile(fans=12000)])
|
||||
await ingest_run(db, first, task, tmp_path)
|
||||
|
||||
second = await _make_run(db, task, started_at=2000)
|
||||
_write_run_dir(tmp_path, [_dy_note("1")], comments=[], subdir="douyin",
|
||||
profiles=[_profile(fans=12300)])
|
||||
await ingest_run(db, second, task, tmp_path)
|
||||
|
||||
stats = list(
|
||||
(
|
||||
await db.scalars(
|
||||
select(MonitorCreatorStat).order_by(MonitorCreatorStat.run_id)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
|
||||
assert [s.fans for s in stats] == [12000, 12300]
|
||||
|
||||
Reference in New Issue
Block a user