butubb
36f41450ae
昵称是**博主级**的属性，而作品是逐条刷的：一个博主只有落在采集窗口里的那几条会被 重刷，掉出窗口的老作品会一直留着旧名字。于是改名之后库里一半新一半旧 —— * 作品栏组头取的是该博主名下作品的 max(name)，**恰好可能取到旧的那个**（比如从 「BBB」改成「AAA」，max 还是 BBB）； * 导出和 SQL 直接看到旧名字。 两处修： 1. ingest 收完这一轮之后，把这一轮看到的 (博主, 昵称) 铺到该博主在本任务下的 **全部**作品上。带上 `!= name` 让没变化的不产生写入。只限本任务 —— 另一个任务里 的同一个人是另一条跟踪线。 2. `list_creators` 的昵称**优先取账号快照**：那一行每次采集都从资料接口重写，是 最新的一份；小红书那条路没有快照，才退回作品上的名字。 顺带把两份 SQL 里的 COALESCE 顺序调成一致（不然查出来还是旧名字）。
394 lines
15 KiB
Python
394 lines
15 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""作品栏里的**博主**：备注（他到底是谁）与账号级指标（他现在多大）。
|
||
|
||
按 creator_hash 分组、显示 creator_name，两样都认不出人：一个是哈希，一个是平台昵称。
|
||
备注是人自己起的名字。账号级指标则是作品列表给不了的东西 —— 作品说的是"这条涨了多少赞"，
|
||
粉丝数说的是"这个人整个账号在涨还是在掉"。
|
||
"""
|
||
|
||
import httpx
|
||
import pytest
|
||
import pytest_asyncio
|
||
|
||
from api.main import app
|
||
from api.monitor import db as monitor_db
|
||
from api.monitor.models import (
|
||
MODE_CREATOR,
|
||
MonitorCreatorStat,
|
||
MonitorNote,
|
||
MonitorTask,
|
||
)
|
||
|
||
CREATOR_HASH = "hash-a"
|
||
NICKNAME = "张三"
|
||
|
||
|
||
@pytest_asyncio.fixture
|
||
async def client(tmp_path):
|
||
monitor_db.set_sqlite_path(tmp_path / "monitor.db")
|
||
await monitor_db.init_db()
|
||
transport = httpx.ASGITransport(app=app)
|
||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as http_client:
|
||
yield http_client
|
||
await monitor_db.dispose_engine()
|
||
|
||
|
||
async def _seed(platform: str = "xhs", task_name: str = "任务") -> int:
|
||
"""一个任务 + 一条作品，博主固定用 CREATOR_HASH / NICKNAME。"""
|
||
async with monitor_db.get_session() as session:
|
||
task = MonitorTask(
|
||
name=task_name, platform=platform, mode=MODE_CREATOR, enabled=True,
|
||
interval_minutes=60, max_notes_count=20, enable_comments=False,
|
||
max_comments_count=50, run_timeout_seconds=3600,
|
||
notify_enabled=False, created_at=0, updated_at=0,
|
||
)
|
||
session.add(task)
|
||
await session.flush()
|
||
session.add(
|
||
MonitorNote(
|
||
task_id=task.id, note_id=f"{platform}-n1", title="作品",
|
||
note_url="", cover="", creator_hash=CREATOR_HASH,
|
||
creator_name=NICKNAME, source_kind="", published_at=None,
|
||
first_seen_run_id=1, first_seen_at=0,
|
||
last_seen_run_id=1, last_seen_at=0,
|
||
)
|
||
)
|
||
return task.id
|
||
|
||
|
||
async def _notes(client, platform: str = "xhs"):
|
||
return (await client.get("/api/monitor/notes", params={"platform": platform})).json()["notes"]
|
||
|
||
|
||
async def _set_alias(client, alias: str, platform: str = "xhs", creator_hash=CREATOR_HASH):
|
||
return await client.put(
|
||
f"/api/monitor/creators/{creator_hash}",
|
||
params={"platform": platform},
|
||
json={"alias": alias},
|
||
)
|
||
|
||
|
||
class TestCreatorAlias:
|
||
@pytest.mark.asyncio
|
||
async def test_notes_start_without_an_alias(self, client):
|
||
await _seed()
|
||
|
||
assert (await _notes(client))[0]["creator_alias"] == ""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_an_alias_comes_back_with_the_notes(self, client):
|
||
await _seed()
|
||
|
||
response = await _set_alias(client, "竞品A")
|
||
|
||
assert response.status_code == 200
|
||
note = (await _notes(client))[0]
|
||
assert note["creator_alias"] == "竞品A"
|
||
# 备注是**叠加**在昵称之上的，不是替换 —— 昵称仍然是有用的对照。
|
||
assert note["creator_name"] == NICKNAME
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_an_alias_is_shared_across_tasks(self, client):
|
||
"""同一个博主出现在两个任务里，备注只该填一次。
|
||
|
||
creator_hash 对同一个 uid 是稳定的，所以键取 (platform, creator_hash) 而不是
|
||
按任务存 —— 否则每加一个任务都要重新认一遍人。
|
||
"""
|
||
await _seed(task_name="任务甲")
|
||
await _seed(task_name="任务乙")
|
||
await _set_alias(client, "竞品A")
|
||
|
||
for note in await _notes(client):
|
||
assert note["creator_alias"] == "竞品A"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_the_alias_does_not_leak_to_another_platform(self, client):
|
||
"""同一个哈希在另一个平台上是另一个（或同一个）人 —— 别串味。"""
|
||
await _seed(platform="xhs")
|
||
await _seed(platform="dy")
|
||
|
||
await _set_alias(client, "小红书那边的", platform="xhs")
|
||
|
||
assert (await _notes(client, "xhs"))[0]["creator_alias"] == "小红书那边的"
|
||
assert (await _notes(client, "dy"))[0]["creator_alias"] == ""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_an_empty_alias_clears_it(self, client):
|
||
await _seed()
|
||
await _set_alias(client, "竞品A")
|
||
|
||
await _set_alias(client, "")
|
||
|
||
assert (await _notes(client))[0]["creator_alias"] == ""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_an_alias_is_trimmed(self, client):
|
||
await _seed()
|
||
|
||
await _set_alias(client, " 竞品A ")
|
||
|
||
assert (await _notes(client))[0]["creator_alias"] == "竞品A"
|
||
|
||
|
||
async def _add_stat(
|
||
task_id: int,
|
||
run_id: int,
|
||
fans: int | None,
|
||
*,
|
||
total_favorited: int | None = 83000,
|
||
works: int | None = 42,
|
||
creator_hash: str = CREATOR_HASH,
|
||
captured_at: int = 1,
|
||
creator_id: str = "",
|
||
nickname: str = NICKNAME,
|
||
) -> None:
|
||
async with monitor_db.get_session() as session:
|
||
session.add(
|
||
MonitorCreatorStat(
|
||
task_id=task_id,
|
||
run_id=run_id,
|
||
creator_hash=creator_hash,
|
||
nickname=nickname,
|
||
creator_id=creator_id,
|
||
fans=fans,
|
||
total_favorited=total_favorited,
|
||
works_count=works,
|
||
following=7,
|
||
captured_at=captured_at,
|
||
)
|
||
)
|
||
|
||
|
||
class TestCreatorStats:
|
||
"""账号级指标跟着作品一起返回 —— 界面上是按博主归组的，为了一个组头再发一轮请求
|
||
没道理。"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_without_snapshots_the_fields_are_null(self, client):
|
||
"""null 而不是 0：0 会显示成「粉丝 0」，而事实是"还没采到"。"""
|
||
await _seed()
|
||
|
||
note = (await _notes(client))[0]
|
||
|
||
assert note["creator_fans"] is None
|
||
assert note["creator_total_favorited"] is None
|
||
assert note["creator_works"] is None
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_a_snapshot_shows_up_on_every_work_of_that_creator(self, client):
|
||
task_id = await _seed()
|
||
await _add_stat(task_id, run_id=1, fans=12000)
|
||
|
||
for note in await _notes(client):
|
||
assert note["creator_fans"] == 12000
|
||
assert note["creator_total_favorited"] == 83000
|
||
assert note["creator_works"] == 42
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_the_latest_run_wins(self, client):
|
||
"""一轮一条，所以总会有好几条 —— 给界面的必须是最近那条。"""
|
||
task_id = await _seed()
|
||
await _add_stat(task_id, run_id=1, fans=12000)
|
||
await _add_stat(task_id, run_id=2, fans=12300)
|
||
|
||
assert (await _notes(client))[0]["creator_fans"] == 12300
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_another_platforms_snapshot_does_not_leak(self, client):
|
||
"""两个平台上恰好同名同哈希的博主是两个人 —— 快照挂在任务上，不该串。"""
|
||
await _seed(platform="xhs")
|
||
dy_task_id = await _seed(platform="dy")
|
||
await _add_stat(dy_task_id, run_id=1, fans=999)
|
||
|
||
assert (await _notes(client, "xhs"))[0]["creator_fans"] is None
|
||
assert (await _notes(client, "dy"))[0]["creator_fans"] == 999
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_an_unparsed_count_stays_null(self, client):
|
||
"""快照在，但某一项没解析出来 —— 那一项必须是 null，不能变成 0。
|
||
|
||
和上一条的区别：那条是"根本没有快照"，这条是"有快照、其中一项平台没给"。
|
||
界面上两种都该是「—」。
|
||
"""
|
||
task_id = await _seed()
|
||
await _add_stat(
|
||
task_id, run_id=1, fans=None, total_favorited=None, works=None
|
||
)
|
||
|
||
note = (await _notes(client))[0]
|
||
|
||
assert note["creator_fans"] is None
|
||
assert note["creator_works"] is None
|
||
# 快照本身是有的（有采集时间），只是值不知道 —— 前端要能分开这两件事。
|
||
assert note["creator_stats_at"] is not None
|
||
|
||
|
||
async def _seed_task(platform: str = "xhs", task_name: str = "空任务") -> int:
|
||
"""只有任务，**一条作品都没有**。"""
|
||
async with monitor_db.get_session() as session:
|
||
task = MonitorTask(
|
||
name=task_name, platform=platform, mode=MODE_CREATOR, enabled=True,
|
||
interval_minutes=60, max_notes_count=20, enable_comments=False,
|
||
max_comments_count=50, run_timeout_seconds=3600,
|
||
notify_enabled=False, created_at=0, updated_at=0,
|
||
)
|
||
session.add(task)
|
||
await session.flush()
|
||
return task.id
|
||
|
||
|
||
async def _creators(client, platform: str = "xhs"):
|
||
return (await client.get("/api/monitor/notes", params={"platform": platform})).json()["creators"]
|
||
|
||
|
||
class TestCreatorList:
|
||
"""作品栏要显示谁 —— **包括一条作品都没有的博主**。
|
||
|
||
组原先是从作品推出来的，于是「目标加了、资料采到了、粉丝数在库里，界面上什么都
|
||
没有」。而这类博主恰恰最该看见：还在涨粉，只是最近没发东西。
|
||
"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_a_creator_with_no_works_still_shows_up(self, client):
|
||
"""**这条就是这个改动的全部理由。**"""
|
||
task_id = await _seed_task()
|
||
await _add_stat(task_id, run_id=1, fans=12000)
|
||
|
||
creators = await _creators(client)
|
||
|
||
assert len(creators) == 1
|
||
assert creators[0]["creator_hash"] == CREATOR_HASH
|
||
assert creators[0]["creator_name"] == NICKNAME # 只剩快照这一个来源
|
||
assert creators[0]["note_count"] == 0
|
||
assert creators[0]["creator_fans"] == 12000
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_a_creator_with_works_carries_their_count(self, client):
|
||
task_id = await _seed()
|
||
await _add_stat(task_id, run_id=1, fans=12000)
|
||
|
||
creators = await _creators(client)
|
||
|
||
assert len(creators) == 1
|
||
assert creators[0]["note_count"] == 1
|
||
assert creators[0]["creator_name"] == NICKNAME
|
||
assert creators[0]["creator_fans"] == 12000
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_a_creator_with_neither_works_nor_a_snapshot_is_absent(self, client):
|
||
"""两个来源都没有 = 我们对他一无所知，不该凭空造一个组出来。"""
|
||
await _seed_task()
|
||
|
||
assert await _creators(client) == []
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_a_work_without_a_snapshot_still_lists_its_creator(self, client):
|
||
"""小红书那条路不产生账号快照 —— 那边只能靠作品认出人来。"""
|
||
await _seed()
|
||
|
||
creators = await _creators(client)
|
||
|
||
assert len(creators) == 1
|
||
assert creators[0]["note_count"] == 1
|
||
assert creators[0]["creator_fans"] is None
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_the_creator_remark_comes_along(self, client):
|
||
"""没有作品的博主也要能起备注 —— 否则「这是谁」在最需要的时候认不出来。"""
|
||
task_id = await _seed_task()
|
||
await _add_stat(task_id, run_id=1, fans=12000)
|
||
|
||
await _set_alias(client, "竞品A")
|
||
|
||
assert (await _creators(client))[0]["creator_alias"] == "竞品A"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_platforms_stay_apart(self, client):
|
||
dy_task = await _seed_task(platform="dy")
|
||
await _add_stat(dy_task, run_id=1, fans=999)
|
||
|
||
assert await _creators(client, "xhs") == []
|
||
assert (await _creators(client, "dy"))[0]["creator_fans"] == 999
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_the_same_creator_under_two_tasks_is_two_rows_the_ui_merges(self, client):
|
||
"""服务端按 任务×博主 给（快照就是那么存的），合并交给界面 —— 因为备注跨任务
|
||
是同一条，合并之后的组才是人眼里的「一个博主」。"""
|
||
first = await _seed(task_name="任务甲")
|
||
second = await _seed(task_name="任务乙")
|
||
await _add_stat(first, run_id=1, fans=12000)
|
||
|
||
creators = await _creators(client)
|
||
|
||
assert len(creators) == 2
|
||
assert {row["creator_hash"] for row in creators} == {CREATOR_HASH}
|
||
assert sum(row["note_count"] for row in creators) == 2
|
||
|
||
|
||
class TestCreatorProfileLink:
|
||
"""组头上那个「跳到他主页」的按钮。
|
||
|
||
它要的是**平台上的主页 id**，不是 ``creator_hash``（哈希单向，拼不出地址）。所以
|
||
这个 id 是采集资料时单独存下来的；拿不到就返回空串让界面不画按钮。
|
||
"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_the_link_is_built_from_the_stored_id(self, client):
|
||
task_id = await _seed_task(platform="dy")
|
||
await _add_stat(task_id, run_id=1, fans=1, creator_id="MS4wLjABAAAAsomebody")
|
||
|
||
creators = await _creators(client, "dy")
|
||
|
||
assert creators[0]["creator_url"] == (
|
||
"https://www.douyin.com/user/MS4wLjABAAAAsomebody"
|
||
)
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_no_id_means_no_link(self, client):
|
||
"""没采到主页 id 就不给链接 —— 画一个点开 404 的按钮比不画糟。"""
|
||
task_id = await _seed_task(platform="dy")
|
||
await _add_stat(task_id, run_id=1, fans=1, creator_id="")
|
||
|
||
assert (await _creators(client, "dy"))[0]["creator_url"] == ""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_a_creator_with_only_works_has_no_link_either(self, client):
|
||
"""小红书那条路根本不落创作者资料，所以那边永远是空的 —— 这是已知的、可接受的。"""
|
||
await _seed(platform="xhs")
|
||
|
||
assert (await _creators(client, "xhs"))[0]["creator_url"] == ""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_a_creator_with_no_works_still_gets_the_link(self, client):
|
||
"""**没有作品的博主照样能跳过去** —— 他恰恰是你想点进去看看发的什么的那个人。"""
|
||
task_id = await _seed_task(platform="dy")
|
||
await _add_stat(task_id, run_id=1, fans=1, creator_id="MS4wLjABAAAAquiet")
|
||
|
||
creators = await _creators(client, "dy")
|
||
|
||
assert creators[0]["note_count"] == 0
|
||
assert creators[0]["creator_url"].endswith("/user/MS4wLjABAAAAquiet")
|
||
|
||
|
||
class TestNicknameSource:
|
||
"""博主改名之后，组头显示哪个名字。
|
||
|
||
**账号快照里的昵称优先**：那一行每次采集都从资料接口重写一遍，是这一行最新的一份。
|
||
作品上的名字只会跟着「这一轮采到的那几条」更新 —— 掉出采集窗口的老作品可能还留着
|
||
旧名字，而作品栏取的是该博主名下作品的 max(name)，恰好可能取到旧的。
|
||
"""
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_the_snapshot_name_wins_over_a_stale_work_name(self, client):
|
||
task_id = await _seed(platform="dy") # 作品上的名字还是 NICKNAME
|
||
await _add_stat(task_id, run_id=1, fans=1, nickname="改过的名字")
|
||
|
||
assert (await _creators(client, "dy"))[0]["creator_name"] == "改过的名字"
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_without_a_snapshot_it_falls_back_to_the_work_name(self, client):
|
||
"""小红书那条路不产生账号快照 —— 只能靠作品上的名字。"""
|
||
await _seed(platform="xhs")
|
||
|
||
assert (await _creators(client, "xhs"))[0]["creator_name"] == NICKNAME
|