feat(monitor): 博主的粉丝数,以及给作品起备注
Deploy VitePress site to Pages / build (push) Canceled after 0s
Deploy VitePress site to Pages / Deploy (push) Canceled after 0s

两件都是「作品栏里把这东西认出来」的延伸:

* **账号级指标**:作品列表只会说「这条涨了多少赞」,说不了「这个人整个
  账号的粉丝在涨还是在掉」。抖音的资料接口本来就有粉丝数/总获赞/作品数,
  每轮顺手记一条快照(`monitor_creator_stat`,粒度 = 任务×博主×轮次,
  和作品指标同形)。组头显示最近一条。

  快照在「一条作品都没采到」的早退**之前**落:作品列表被风控挡住的那一轮,
  正是「粉丝还在涨、但新作品没在发现」最该被看见的时刻。

* **作品备注**:博主备注回答「这个账号是谁」,这条回答「这条我要盯着」。
  一个博主底下常常只有一两件值得盯的作品,所以不能合并成一条。键取
  (platform, note_id),跨任务共用一份。

两边都守住同一条口径:**不知道就是 null,不写成 0** —— 0 在趋势图上是一条
砸到底的线,和「还没采到」是两回事。
This commit is contained in:
2026-10-10 18:09:03 +08:00
parent 0a88474c92
commit 20e672834c
15 changed files with 953 additions and 10 deletions
+137
View File
@@ -13,6 +13,35 @@ import pytest
from api.monitor import douyin_api, douyin_fetch
def _profile(**overrides) -> dict:
profile = {
"creator_hash": "hash",
"nickname": "博主",
"unique_id": "abc",
"fans": 12000,
"total_favorited": 83000,
"works": 42,
"following": 7,
}
profile.update(overrides)
return profile
@pytest.fixture(autouse=True)
def fake_profile(monkeypatch):
"""每个用例都挡住「问博主资料」这一跳。
它是附加信息,不在任何一条编排路径上,但真发出去就会去连 9222 那个浏览器 ——
于是所有 creator 用例都会多出一次连接失败、并把 ``errors`` 弄脏。想验它自己的
用例再单独覆盖这个 fixture。
"""
async def _profile_call(sec_user_id, *, cookie=""):
return _profile()
monkeypatch.setattr(douyin_api, "author_profile", _profile_call)
def _video(aweme_id: str, likes: str = "1") -> dict:
return {
"aweme_id": aweme_id,
@@ -272,3 +301,111 @@ class TestDegradation:
# 评论拿不到是小事,作品不能跟着丢。
assert result["notes"] == 1
assert any("评论失败" in error for error in result["errors"])
class TestCreatorProfile:
"""博主的**账号级**指标 —— 粉丝 / 总获赞 / 作品数。
作品列表给不了这个东西:它说的是一件作品涨了多少赞,不是这个人整个账号的粉丝
在涨还是在掉。单独问一次资料接口。
"""
@staticmethod
def _profiles(tmp_path):
files = list((tmp_path / "douyin" / "jsonl").glob("*_profile_*.jsonl"))
assert len(files) == 1
return _read(files[0])
@pytest.mark.asyncio
async def test_the_profile_lands_in_the_run_dir(self, monkeypatch, tmp_path):
async def fake_videos(sec_user_id, count=20, *, cookie=""):
return [_video("111")]
monkeypatch.setattr(douyin_api, "author_videos", fake_videos)
await _collect(tmp_path, want_comments=False)
assert self._profiles(tmp_path) == [_profile()]
@pytest.mark.asyncio
async def test_the_profile_is_keyed_by_the_same_hash_as_the_works(
self, monkeypatch, tmp_path
):
"""**这条是关键。** 快照表的唯一键是 (任务, creator_hash, 轮次),而界面上是按
作品的 creator_hash 归组去查它的。两边只要差一个字符,粉丝数就永远查不出来 ——
而且是静默的:表里有数据,界面上什么都没有。
"""
async def fake_videos(sec_user_id, count=20, *, cookie=""):
return [_video("111")] # 作品带的哈希是 "hash"
# 资料接口自己算出来的是另一个值(比如它那边 uid 缺字段、只能拿 sec_uid 算)。
async def off_hash_profile(sec_user_id, *, cookie=""):
return _profile(creator_hash="另一个哈希")
monkeypatch.setattr(douyin_api, "author_videos", fake_videos)
monkeypatch.setattr(douyin_api, "author_profile", off_hash_profile)
await _collect(tmp_path, want_comments=False)
# 以作品为准:作品才是界面上的行,快照必须挂在能查到它的那个键上。
assert self._profiles(tmp_path)[0]["creator_hash"] == "hash"
@pytest.mark.asyncio
async def test_a_profile_without_an_identity_is_dropped(self, monkeypatch, tmp_path):
"""哈希都算不出来的快照,落下去只会是一条谁也查不到的垃圾。"""
async def fake_videos(sec_user_id, count=20, *, cookie=""):
return []
async def anonymous_profile(sec_user_id, *, cookie=""):
return _profile(creator_hash="")
monkeypatch.setattr(douyin_api, "author_videos", fake_videos)
monkeypatch.setattr(douyin_api, "author_profile", anonymous_profile)
result = await _collect(tmp_path, want_comments=False)
assert self._profiles(tmp_path) == []
assert any("身份标识" in error for error in result["errors"])
@pytest.mark.asyncio
async def test_a_failing_profile_does_not_lose_the_works(self, monkeypatch, tmp_path):
"""附加信息拿不到,这一轮采到的作品不能跟着判成失败。"""
async def fake_videos(sec_user_id, count=20, *, cookie=""):
return [_video("111")]
async def broken_profile(sec_user_id, *, cookie=""):
raise douyin_api.DouyinApiError("资料接口抽风")
monkeypatch.setattr(douyin_api, "author_videos", fake_videos)
monkeypatch.setattr(douyin_api, "author_profile", broken_profile)
result = await _collect(tmp_path, want_comments=False)
assert result["notes"] == 1
assert self._profiles(tmp_path) == []
assert any("资料失败" in error for error in result["errors"])
@pytest.mark.asyncio
async def test_a_work_target_never_asks_for_a_profile(self, monkeypatch, tmp_path):
"""作品模式的目标是一件作品,没有「这个博主是谁」可问 —— 不该白发一个请求。"""
asked = []
async def fake_detail(aweme_id, *, cookie=""):
return _video(aweme_id)
async def recording_profile(sec_user_id, *, cookie=""):
asked.append(sec_user_id)
return _profile()
monkeypatch.setattr(douyin_api, "video_detail", fake_detail)
monkeypatch.setattr(douyin_api, "author_profile", recording_profile)
await _collect(tmp_path, mode="note", want_comments=False)
assert asked == []
# 文件仍然建出来(空的):事后翻 run 目录能看出「这次根本没问过」。
assert self._profiles(tmp_path) == []
+99 -3
View File
@@ -1,8 +1,9 @@
# -*- coding: utf-8 -*-
"""博主备注 —— 作品栏里那一位到底是谁。
"""作品栏里的**博主**:备注(他到底是谁)与账号级指标(他现在多大)。
按 creator_hash 分组、显示 creator_name,两样都认不出人:一个是哈希,一个是平台昵称。
备注是人自己起的名字。
备注是人自己起的名字。账号级指标则是作品列表给不了的东西 —— 作品说的是"这条涨了多少赞",
粉丝数说的是"这个人整个账号在涨还是在掉"。
"""
import httpx
@@ -11,7 +12,12 @@ import pytest_asyncio
from api.main import app
from api.monitor import db as monitor_db
from api.monitor.models import MODE_CREATOR, MonitorNote, MonitorTask
from api.monitor.models import (
MODE_CREATOR,
MonitorCreatorStat,
MonitorNote,
MonitorTask,
)
CREATOR_HASH = "hash-a"
NICKNAME = "张三"
@@ -122,3 +128,93 @@ class TestCreatorAlias:
await _set_alias(client, " 竞品A ")
assert (await _notes(client))[0]["creator_alias"] == "竞品A"
async def _add_stat(
task_id: int,
run_id: int,
fans: int | None,
*,
total_favorited: int | None = 83000,
works: int | None = 42,
creator_hash: str = CREATOR_HASH,
captured_at: int = 1,
) -> None:
async with monitor_db.get_session() as session:
session.add(
MonitorCreatorStat(
task_id=task_id,
run_id=run_id,
creator_hash=creator_hash,
nickname=NICKNAME,
fans=fans,
total_favorited=total_favorited,
works_count=works,
following=7,
captured_at=captured_at,
)
)
class TestCreatorStats:
"""账号级指标跟着作品一起返回 —— 界面上是按博主归组的,为了一个组头再发一轮请求
没道理。"""
@pytest.mark.asyncio
async def test_without_snapshots_the_fields_are_null(self, client):
"""null 而不是 0:0 会显示成「粉丝 0」,而事实是"还没采到"。"""
await _seed()
note = (await _notes(client))[0]
assert note["creator_fans"] is None
assert note["creator_total_favorited"] is None
assert note["creator_works"] is None
@pytest.mark.asyncio
async def test_a_snapshot_shows_up_on_every_work_of_that_creator(self, client):
task_id = await _seed()
await _add_stat(task_id, run_id=1, fans=12000)
for note in await _notes(client):
assert note["creator_fans"] == 12000
assert note["creator_total_favorited"] == 83000
assert note["creator_works"] == 42
@pytest.mark.asyncio
async def test_the_latest_run_wins(self, client):
"""一轮一条,所以总会有好几条 —— 给界面的必须是最近那条。"""
task_id = await _seed()
await _add_stat(task_id, run_id=1, fans=12000)
await _add_stat(task_id, run_id=2, fans=12300)
assert (await _notes(client))[0]["creator_fans"] == 12300
@pytest.mark.asyncio
async def test_another_platforms_snapshot_does_not_leak(self, client):
"""两个平台上恰好同名同哈希的博主是两个人 —— 快照挂在任务上,不该串。"""
await _seed(platform="xhs")
dy_task_id = await _seed(platform="dy")
await _add_stat(dy_task_id, run_id=1, fans=999)
assert (await _notes(client, "xhs"))[0]["creator_fans"] is None
assert (await _notes(client, "dy"))[0]["creator_fans"] == 999
@pytest.mark.asyncio
async def test_an_unparsed_count_stays_null(self, client):
"""快照在,但某一项没解析出来 —— 那一项必须是 null,不能变成 0。
和上一条的区别:那条是"根本没有快照",这条是"有快照、其中一项平台没给"。
界面上两种都该是「—」。
"""
task_id = await _seed()
await _add_stat(
task_id, run_id=1, fans=None, total_favorited=None, works=None
)
note = (await _notes(client))[0]
assert note["creator_fans"] is None
assert note["creator_works"] is None
# 快照本身是有的(有采集时间),只是值不知道 —— 前端要能分开这两件事。
assert note["creator_stats_at"] is not None
+147
View File
@@ -54,6 +54,7 @@ from api.monitor.models import (
MODE_CREATOR,
MonitorBase,
MonitorComment,
MonitorCreatorStat,
MonitorEvent,
MonitorNote,
MonitorNoteMetric,
@@ -127,11 +128,15 @@ def _write_run_dir(
notes: List[Dict[str, Any]],
comments: Optional[List[Dict[str, Any]]] = None,
subdir: str = "xhs",
profiles: Optional[List[Dict[str, Any]]] = None,
) -> Path:
"""Write a run's jsonl output in the crawler's own layout.
``subdir`` 是**爬虫**落盘的目录名,不是监控层的平台 id —— 抖音那边这两者不同
(平台 id 是 ``dy``、目录是 ``douyin``),所以必须能分开指定,否则测不出那个差异。
``profiles`` 为 None 时**不写**这个文件(小红书那条路根本不产生它),给列表时写
——包括空列表,那是「问了但没问到」。
"""
jsonl_dir = root / subdir / "jsonl"
jsonl_dir.mkdir(parents=True, exist_ok=True)
@@ -147,6 +152,12 @@ def _write_run_dir(
"\n".join(json.dumps(c, ensure_ascii=False) for c in comments),
encoding="utf-8",
)
if profiles is not None:
profile_file = jsonl_dir / "creator_profile_2026-01-01.jsonl"
profile_file.write_text(
"\n".join(json.dumps(p, ensure_ascii=False) for p in profiles),
encoding="utf-8",
)
return root
@@ -829,3 +840,139 @@ class TestFailureDiagnosis:
events = await _events(db, EVENT_RUN_FAILED)
assert "account blocked" in events[0].title
# --------------------------------------------------------------------------
# 博主账号级指标(粉丝 / 总获赞 / 作品数)
# --------------------------------------------------------------------------
def _profile(creator_hash: str = "hash", **extra) -> Dict[str, Any]:
"""``creator_profile_*.jsonl`` 里的一行 —— 形状由 douyin_api.author_profile 决定。"""
record: Dict[str, Any] = {
"creator_hash": creator_hash,
"nickname": "博主",
"unique_id": "abc",
"fans": 12000,
"total_favorited": 83000,
"works": 42,
"following": 7,
}
record.update(extra)
return record
class TestCreatorStatSnapshots:
"""**账号级**指标和作品级指标是两回事:后者说"这条视频涨了多少赞",前者说
"这个人整个账号的粉丝在涨还是在掉"。作品列表给不了后者,所以单独存一张表。
"""
async def _ingest(self, db, tmp_path, notes, profiles, platform="dy", subdir="douyin"):
task = await _make_task(db, platform=platform)
run = await _make_run(db, task, started_at=1)
_write_run_dir(tmp_path, notes, comments=[], subdir=subdir, profiles=profiles)
result = await ingest_run(db, run, task, tmp_path)
return task, run, result
@pytest.mark.asyncio
async def test_a_profile_becomes_a_snapshot(self, db, tmp_path):
task, run, _result = await self._ingest(db, tmp_path, [_dy_note("1")], [_profile()])
stat = await db.scalar(select(MonitorCreatorStat))
assert stat is not None
assert (stat.task_id, stat.run_id) == (task.id, run.id)
assert stat.creator_hash == "hash"
assert stat.nickname == "博主"
assert (stat.fans, stat.total_favorited, stat.works_count) == (12000, 83000, 42)
@pytest.mark.asyncio
async def test_a_missing_count_stays_null_not_zero(self, db, tmp_path):
"""0 是真实值(掉到零),null 是不知道。混起来趋势图就是在撒谎。"""
await self._ingest(db, tmp_path, [_dy_note("1")], [_profile(fans=None)])
stat = await db.scalar(select(MonitorCreatorStat))
assert stat.fans is None
# 同一个博主其它字段照常。
assert stat.works_count == 42
@pytest.mark.asyncio
async def test_abbreviated_counts_are_parsed(self, db, tmp_path):
"""走的是和作品指标同一个 parse_count —— 平台给你「1.2万」也得认。"""
await self._ingest(db, tmp_path, [_dy_note("1")], [_profile(fans="1.2万")])
assert (await db.scalar(select(MonitorCreatorStat))).fans == 12000
@pytest.mark.asyncio
async def test_the_same_creator_twice_in_one_run_yields_one_snapshot(self, db, tmp_path):
"""一个任务可以配多个目标,退化路径下它们可能落在同一个博主身上。
唯一键是 (task_id, creator_hash, run_id) —— 重复插入会撞键,把整轮炸掉。
(和作品重复那次是同一类事故。)
"""
await self._ingest(
db, tmp_path, [_dy_note("1")], [_profile(), _profile(nickname="另一条")]
)
stats = list((await db.scalars(select(MonitorCreatorStat))).all())
assert len(stats) == 1
@pytest.mark.asyncio
async def test_a_profile_without_a_hash_is_skipped(self, db, tmp_path):
"""哈希都算不出来,这条快照谁也查不到,落下去只是垃圾。"""
await self._ingest(db, tmp_path, [_dy_note("1")], [_profile(creator_hash="")])
assert (await db.scalars(select(MonitorCreatorStat))).all() == []
@pytest.mark.asyncio
async def test_no_profile_file_is_fine(self, db, tmp_path):
"""小红书那条路(爬虫进程)根本不产生这个文件 —— 不能因此报错。"""
task = await _make_task(db) # xhs
run = await _make_run(db, task, started_at=1)
_write_run_dir(tmp_path, [_note("n1")], comments=[]) # 没有 profiles 参数
result = await ingest_run(db, run, task, tmp_path)
assert result.status == RUN_SUCCESS
assert (await db.scalars(select(MonitorCreatorStat))).all() == []
@pytest.mark.asyncio
async def test_the_stats_are_kept_even_when_no_works_were_fetched(self, db, tmp_path):
"""**这条是这里最值得留的一个。**
作品列表被风控挡住时,这一轮一条作品都拿不到、run 会被判成失败。但博主的粉丝数
并不因为这件事就不存在 —— 「粉丝还在涨,但新作品没在发现」恰恰是最该看见的时刻。
快照要是挂在「作品采到了」后面,就正好在最需要它的那一轮丢掉。
"""
_task, run, result = await self._ingest(db, tmp_path, [], [_profile()])
assert result.status == RUN_PARTIAL # 一条作品都没有,这轮确实不算成功
assert run.status == RUN_PARTIAL
stat = await db.scalar(select(MonitorCreatorStat))
assert stat is not None and stat.fans == 12000
@pytest.mark.asyncio
async def test_each_run_adds_its_own_snapshot(self, db, tmp_path):
"""趋势靠的就是这个:一条一轮,不要覆盖。"""
task = await _make_task(db, platform="dy")
first = await _make_run(db, task, started_at=1)
_write_run_dir(tmp_path, [_dy_note("1")], comments=[], subdir="douyin",
profiles=[_profile(fans=12000)])
await ingest_run(db, first, task, tmp_path)
second = await _make_run(db, task, started_at=2000)
_write_run_dir(tmp_path, [_dy_note("1")], comments=[], subdir="douyin",
profiles=[_profile(fans=12300)])
await ingest_run(db, second, task, tmp_path)
stats = list(
(
await db.scalars(
select(MonitorCreatorStat).order_by(MonitorCreatorStat.run_id)
)
).all()
)
assert [s.fans for s in stats] == [12000, 12300]
+146
View File
@@ -0,0 +1,146 @@
# -*- coding: utf-8 -*-
"""作品备注 —— 一个博主底下,哪几条是真正要盯的。
和博主备注(test_monitor_creators.py)是一对,但回答的不是同一个问题:博主备注回答
「这个账号是谁」,作品备注回答「这条作品我要盯着」。一个博主底下常常只有一两件值得
盯的作品,所以不能合成一条。
"""
import httpx
import pytest
import pytest_asyncio
from api.main import app
from api.monitor import db as monitor_db
from api.monitor.models import MODE_CREATOR, MonitorNote, MonitorTask
NOTE_ID = "note-a"
TITLE = "中秋哪儿都堵"
@pytest_asyncio.fixture
async def client(tmp_path):
monitor_db.set_sqlite_path(tmp_path / "monitor.db")
await monitor_db.init_db()
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as http_client:
yield http_client
await monitor_db.dispose_engine()
async def _seed(platform: str = "xhs", task_name: str = "任务", note_id: str = NOTE_ID) -> int:
async with monitor_db.get_session() as session:
task = MonitorTask(
name=task_name, platform=platform, mode=MODE_CREATOR, enabled=True,
interval_minutes=60, max_notes_count=20, enable_comments=False,
max_comments_count=50, run_timeout_seconds=3600,
notify_enabled=False, created_at=0, updated_at=0,
)
session.add(task)
await session.flush()
session.add(
MonitorNote(
task_id=task.id, note_id=note_id, title=TITLE,
note_url="", cover="", creator_hash="hash-a",
creator_name="张三", source_kind="", published_at=None,
first_seen_run_id=1, first_seen_at=0,
last_seen_run_id=1, last_seen_at=0,
)
)
return task.id
async def _notes(client, platform: str = "xhs"):
return (await client.get("/api/monitor/notes", params={"platform": platform})).json()["notes"]
async def _set_alias(client, alias: str, platform: str = "xhs", note_id: str = NOTE_ID):
return await client.put(
f"/api/monitor/notes/{note_id}",
params={"platform": platform},
json={"alias": alias},
)
class TestNoteAlias:
@pytest.mark.asyncio
async def test_notes_start_without_a_remark(self, client):
await _seed()
assert (await _notes(client))[0]["note_alias"] == ""
@pytest.mark.asyncio
async def test_a_remark_comes_back_with_the_notes(self, client):
await _seed()
response = await _set_alias(client, "重点")
assert response.status_code == 200
note = (await _notes(client))[0]
assert note["note_alias"] == "重点"
# 备注是**叠加**在标题之上的,不是替换 —— 标题仍然是这条作品本身。
assert note["title"] == TITLE
@pytest.mark.asyncio
async def test_a_remark_is_shared_across_tasks(self, client):
"""同一件作品被两个任务都监控时,备注只该填一次。"""
await _seed(task_name="任务甲")
await _seed(task_name="任务乙")
await _set_alias(client, "重点")
for note in await _notes(client):
assert note["note_alias"] == "重点"
@pytest.mark.asyncio
async def test_a_remark_does_not_leak_to_another_platform(self, client):
"""作品的 id 是平台各自的编号体系 —— 抖音的 123 和小红书的 123 是两条作品。"""
await _seed(platform="xhs")
await _seed(platform="dy")
await _set_alias(client, "小红书那边的", platform="xhs")
assert (await _notes(client, "xhs"))[0]["note_alias"] == "小红书那边的"
assert (await _notes(client, "dy"))[0]["note_alias"] == ""
@pytest.mark.asyncio
async def test_a_remark_does_not_leak_to_another_work(self, client):
"""钉住这里的**作用域**:键是 note_id。写错成按任务存的话,给一条起了备注,
同一个博主底下的其它作品会跟着一起变 —— 那这个功能就没用了。"""
await _seed(note_id="note-a")
await _seed(task_name="另一个任务", note_id="note-b")
await _set_alias(client, "重点", note_id="note-a")
by_id = {note["note_id"]: note["note_alias"] for note in await _notes(client)}
assert by_id == {"note-a": "重点", "note-b": ""}
@pytest.mark.asyncio
async def test_an_empty_remark_clears_it(self, client):
await _seed()
await _set_alias(client, "重点")
await _set_alias(client, "")
assert (await _notes(client))[0]["note_alias"] == ""
@pytest.mark.asyncio
async def test_a_remark_is_trimmed(self, client):
await _seed()
await _set_alias(client, " 重点 ")
assert (await _notes(client))[0]["note_alias"] == "重点"
@pytest.mark.asyncio
async def test_the_two_kinds_of_remark_stay_apart(self, client):
"""博主备注和作品备注是两张表、两个键 —— 一个不该把另一个盖掉。"""
await _seed()
await _set_alias(client, "重点")
await client.put(
"/api/monitor/creators/hash-a", params={"platform": "xhs"}, json={"alias": "竞品A"}
)
note = (await _notes(client))[0]
assert note["note_alias"] == "重点"
assert note["creator_alias"] == "竞品A"