fix(monitor): 博主改名之后,旧作品上的名字也跟着改
昵称是**博主级**的属性,而作品是逐条刷的:一个博主只有落在采集窗口里的那几条会被 重刷,掉出窗口的老作品会一直留着旧名字。于是改名之后库里一半新一半旧 —— * 作品栏组头取的是该博主名下作品的 max(name),**恰好可能取到旧的那个**(比如从 「BBB」改成「AAA」,max 还是 BBB); * 导出和 SQL 直接看到旧名字。 两处修: 1. ingest 收完这一轮之后,把这一轮看到的 (博主, 昵称) 铺到该博主在本任务下的 **全部**作品上。带上 `!= name` 让没变化的不产生写入。只限本任务 —— 另一个任务里 的同一个人是另一条跟踪线。 2. `list_creators` 的昵称**优先取账号快照**:那一行每次采集都从资料接口重写,是 最新的一份;小红书那条路没有快照,才退回作品上的名字。 顺带把两份 SQL 里的 COALESCE 顺序调成一致(不然查出来还是旧名字)。
This commit is contained in:
+25
-1
@@ -40,7 +40,7 @@ from dataclasses import dataclass, field
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, List, Optional, Sequence
|
from typing import Any, Dict, List, Optional, Sequence
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select, update
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from tools.time_util import get_current_timestamp
|
from tools.time_util import get_current_timestamp
|
||||||
@@ -359,6 +359,10 @@ async def _ingest_notes(
|
|||||||
# 同一件作品在一轮里进来两次会让第二次插入直接撞键、整个 run 崩掉 —— 产物里重复并不
|
# 同一件作品在一轮里进来两次会让第二次插入直接撞键、整个 run 崩掉 —— 产物里重复并不
|
||||||
# 罕见(多个目标指向同一个人、或退化路径重复刷新)。
|
# 罕见(多个目标指向同一个人、或退化路径重复刷新)。
|
||||||
seen_in_run: set = set()
|
seen_in_run: set = set()
|
||||||
|
# 这一轮看到的 (博主哈希 -> 昵称)。**昵称是博主级的属性,而作品是逐条刷的** ——
|
||||||
|
# 一个博主只有落在采集窗口里的那几条会被重刷,掉出窗口的那些会一直留着旧名字。
|
||||||
|
# 所以收完这一轮之后,再拿它把该博主在本任务下的作品统一改一遍(见循环之后)。
|
||||||
|
fresh_names: Dict[str, str] = {}
|
||||||
|
|
||||||
for record in records:
|
for record in records:
|
||||||
note_id = adapter.note_field(record, "note_id")
|
note_id = adapter.note_field(record, "note_id")
|
||||||
@@ -377,6 +381,9 @@ async def _ingest_notes(
|
|||||||
|
|
||||||
title = (adapter.note_field(record, "title") or "")[:500]
|
title = (adapter.note_field(record, "title") or "")[:500]
|
||||||
cover = adapter.cover(record)
|
cover = adapter.cover(record)
|
||||||
|
fresh_name = adapter.note_field(record, "creator_name") or ""
|
||||||
|
if fresh_name:
|
||||||
|
fresh_names[adapter.note_field(record, "creator_hash") or ""] = fresh_name
|
||||||
|
|
||||||
if note is None:
|
if note is None:
|
||||||
note = MonitorNote(
|
note = MonitorNote(
|
||||||
@@ -430,6 +437,23 @@ async def _ingest_notes(
|
|||||||
|
|
||||||
await _snapshot_metrics(session, run, note_id, record, now, is_baseline)
|
await _snapshot_metrics(session, run, note_id, record, now, is_baseline)
|
||||||
|
|
||||||
|
# 把这一轮的昵称铺到该博主的**全部**作品上,不只这一轮采到的那几条。
|
||||||
|
# 不做的话,博主改名之后库里会一半是新名字、一半是旧名字:界面按作品名分组时取的是
|
||||||
|
# 其中任意一个(max),导出和 SQL 更是直接看到旧名字。带上 `!= name` 是为了让没变化
|
||||||
|
# 的那些不产生写入。
|
||||||
|
for creator_hash, name in fresh_names.items():
|
||||||
|
if not creator_hash:
|
||||||
|
continue
|
||||||
|
await session.execute(
|
||||||
|
update(MonitorNote)
|
||||||
|
.where(
|
||||||
|
MonitorNote.task_id == run.task_id,
|
||||||
|
MonitorNote.creator_hash == creator_hash,
|
||||||
|
MonitorNote.creator_name != name,
|
||||||
|
)
|
||||||
|
.values(creator_name=name)
|
||||||
|
)
|
||||||
|
|
||||||
return new_count
|
return new_count
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -945,10 +945,13 @@ async def list_creators(
|
|||||||
{
|
{
|
||||||
"task_id": row_task,
|
"task_id": row_task,
|
||||||
"creator_hash": creator_hash,
|
"creator_hash": creator_hash,
|
||||||
# 昵称优先取作品的(那是界面上本来就在用的),快照的兜底 —— 没有作品
|
# 昵称**优先取账号快照**里的。快照每次采集都从资料接口重写一遍,是这一行
|
||||||
# 的博主只剩快照这一个来源。
|
# 最新的一份;作品上的名字只会跟着「这一轮采到的那几条」更新,掉出采集
|
||||||
"creator_name": (work_row or {}).get("creator_name")
|
# 窗口的老作品可能还留着旧名字。博主改名之后取作品的会显示旧名。
|
||||||
or (stat.nickname if stat else ""),
|
# 没有快照的(小红书那条路不产生快照)才退回作品上的名字。
|
||||||
|
"creator_name": (stat.nickname if stat else "")
|
||||||
|
or (work_row or {}).get("creator_name")
|
||||||
|
or "",
|
||||||
"creator_alias": aliases.get(
|
"creator_alias": aliases.get(
|
||||||
(task_platform.get(row_task, ""), creator_hash), ""
|
(task_platform.get(row_task, ""), creator_hash), ""
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -20,7 +20,7 @@
|
|||||||
SELECT
|
SELECT
|
||||||
acct.creator_hash AS 博主哈希,
|
acct.creator_hash AS 博主哈希,
|
||||||
ca.alias AS 博主备注,
|
ca.alias AS 博主备注,
|
||||||
COALESCE(n.creator_name, st.nickname, '') AS 博主昵称,
|
COALESCE(st.nickname, n.creator_name, '') AS 博主昵称,
|
||||||
st.fans AS 粉丝数,
|
st.fans AS 粉丝数,
|
||||||
st.total_favorited AS 总获赞,
|
st.total_favorited AS 总获赞,
|
||||||
st.works_count AS 主页作品数,
|
st.works_count AS 主页作品数,
|
||||||
|
|||||||
@@ -23,7 +23,7 @@
|
|||||||
SELECT
|
SELECT
|
||||||
acct.creator_hash AS 博主哈希,
|
acct.creator_hash AS 博主哈希,
|
||||||
ca.alias AS 博主备注,
|
ca.alias AS 博主备注,
|
||||||
COALESCE(n.creator_name, st.nickname, '') AS 博主昵称,
|
COALESCE(st.nickname, n.creator_name, '') AS 博主昵称,
|
||||||
st.fans AS 粉丝数,
|
st.fans AS 粉丝数,
|
||||||
st.total_favorited AS 总获赞,
|
st.total_favorited AS 总获赞,
|
||||||
st.works_count AS 主页作品数,
|
st.works_count AS 主页作品数,
|
||||||
|
|||||||
@@ -140,6 +140,7 @@ async def _add_stat(
|
|||||||
creator_hash: str = CREATOR_HASH,
|
creator_hash: str = CREATOR_HASH,
|
||||||
captured_at: int = 1,
|
captured_at: int = 1,
|
||||||
creator_id: str = "",
|
creator_id: str = "",
|
||||||
|
nickname: str = NICKNAME,
|
||||||
) -> None:
|
) -> None:
|
||||||
async with monitor_db.get_session() as session:
|
async with monitor_db.get_session() as session:
|
||||||
session.add(
|
session.add(
|
||||||
@@ -147,7 +148,7 @@ async def _add_stat(
|
|||||||
task_id=task_id,
|
task_id=task_id,
|
||||||
run_id=run_id,
|
run_id=run_id,
|
||||||
creator_hash=creator_hash,
|
creator_hash=creator_hash,
|
||||||
nickname=NICKNAME,
|
nickname=nickname,
|
||||||
creator_id=creator_id,
|
creator_id=creator_id,
|
||||||
fans=fans,
|
fans=fans,
|
||||||
total_favorited=total_favorited,
|
total_favorited=total_favorited,
|
||||||
@@ -367,3 +368,26 @@ class TestCreatorProfileLink:
|
|||||||
|
|
||||||
assert creators[0]["note_count"] == 0
|
assert creators[0]["note_count"] == 0
|
||||||
assert creators[0]["creator_url"].endswith("/user/MS4wLjABAAAAquiet")
|
assert creators[0]["creator_url"].endswith("/user/MS4wLjABAAAAquiet")
|
||||||
|
|
||||||
|
|
||||||
|
class TestNicknameSource:
|
||||||
|
"""博主改名之后,组头显示哪个名字。
|
||||||
|
|
||||||
|
**账号快照里的昵称优先**:那一行每次采集都从资料接口重写一遍,是这一行最新的一份。
|
||||||
|
作品上的名字只会跟着「这一轮采到的那几条」更新 —— 掉出采集窗口的老作品可能还留着
|
||||||
|
旧名字,而作品栏取的是该博主名下作品的 max(name),恰好可能取到旧的。
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_the_snapshot_name_wins_over_a_stale_work_name(self, client):
|
||||||
|
task_id = await _seed(platform="dy") # 作品上的名字还是 NICKNAME
|
||||||
|
await _add_stat(task_id, run_id=1, fans=1, nickname="改过的名字")
|
||||||
|
|
||||||
|
assert (await _creators(client, "dy"))[0]["creator_name"] == "改过的名字"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_without_a_snapshot_it_falls_back_to_the_work_name(self, client):
|
||||||
|
"""小红书那条路不产生账号快照 —— 只能靠作品上的名字。"""
|
||||||
|
await _seed(platform="xhs")
|
||||||
|
|
||||||
|
assert (await _creators(client, "xhs"))[0]["creator_name"] == NICKNAME
|
||||||
|
|||||||
@@ -1010,3 +1010,82 @@ class TestCreatorIdIsStored:
|
|||||||
await ingest_run(db, run, task, tmp_path)
|
await ingest_run(db, run, task, tmp_path)
|
||||||
|
|
||||||
assert (await db.scalar(select(MonitorCreatorStat))).creator_id == ""
|
assert (await db.scalar(select(MonitorCreatorStat))).creator_id == ""
|
||||||
|
|
||||||
|
|
||||||
|
class TestNicknameRefresh:
|
||||||
|
"""博主改名之后,库里不能一半是新名字、一半是旧名字。
|
||||||
|
|
||||||
|
昵称是**博主级**的属性,而作品是逐条刷的 —— 一个博主只有落在采集窗口里的那几条会被
|
||||||
|
重刷,掉出窗口的老作品会一直留着旧名字。界面按作品名分组时取的是其中任意一个,
|
||||||
|
导出和 SQL 更是直接看到旧名字。
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_rename_reaches_the_works_outside_this_runs_window(self, db, tmp_path):
|
||||||
|
task = await _make_task(db, platform="dy")
|
||||||
|
|
||||||
|
# 第一轮:两条作品,都叫「旧名字」。
|
||||||
|
first = await _make_run(db, task, started_at=1)
|
||||||
|
_write_run_dir(
|
||||||
|
tmp_path,
|
||||||
|
[_dy_note("111", nickname="旧名字"), _dy_note("222", nickname="旧名字")],
|
||||||
|
comments=[],
|
||||||
|
subdir="douyin",
|
||||||
|
)
|
||||||
|
await ingest_run(db, first, task, tmp_path)
|
||||||
|
|
||||||
|
# 第二轮:只采到 111(222 掉出了采集窗口),而博主已经改名。
|
||||||
|
second = await _make_run(db, task, started_at=2000)
|
||||||
|
_write_run_dir(
|
||||||
|
tmp_path,
|
||||||
|
[_dy_note("111", nickname="新名字")],
|
||||||
|
comments=[],
|
||||||
|
subdir="douyin",
|
||||||
|
)
|
||||||
|
await ingest_run(db, second, task, tmp_path)
|
||||||
|
|
||||||
|
names = {
|
||||||
|
row.note_id: row.creator_name
|
||||||
|
for row in (await db.scalars(select(MonitorNote))).all()
|
||||||
|
}
|
||||||
|
assert names == {"111": "新名字", "222": "新名字"}
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_the_rename_does_not_leak_into_another_task(self, db, tmp_path):
|
||||||
|
"""博主级不等于全局 —— 另一个任务里的同一个人是另一条跟踪线,别顺手改它。"""
|
||||||
|
mine = await _make_task(db, name="我的", platform="dy")
|
||||||
|
other = await _make_task(db, name="别人的", platform="dy")
|
||||||
|
|
||||||
|
run = await _make_run(db, other, started_at=1)
|
||||||
|
_write_run_dir(tmp_path, [_dy_note("999", nickname="别人的名字")], comments=[], subdir="douyin")
|
||||||
|
await ingest_run(db, run, other, tmp_path)
|
||||||
|
|
||||||
|
run = await _make_run(db, mine, started_at=1)
|
||||||
|
_write_run_dir(tmp_path, [_dy_note("111", nickname="旧名字")], comments=[], subdir="douyin")
|
||||||
|
await ingest_run(db, run, mine, tmp_path)
|
||||||
|
|
||||||
|
run = await _make_run(db, mine, started_at=2000)
|
||||||
|
_write_run_dir(tmp_path, [_dy_note("111", nickname="新名字")], comments=[], subdir="douyin")
|
||||||
|
await ingest_run(db, run, mine, tmp_path)
|
||||||
|
|
||||||
|
by_task = {
|
||||||
|
row.note_id: row.creator_name
|
||||||
|
for row in (await db.scalars(select(MonitorNote))).all()
|
||||||
|
}
|
||||||
|
assert by_task["999"] == "别人的名字"
|
||||||
|
assert by_task["111"] == "新名字"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_an_empty_name_does_not_wipe_the_stored_one(self, db, tmp_path):
|
||||||
|
"""平台这一轮没给昵称时,不要把已有的抹掉。"""
|
||||||
|
task = await _make_task(db, platform="dy")
|
||||||
|
|
||||||
|
run = await _make_run(db, task, started_at=1)
|
||||||
|
_write_run_dir(tmp_path, [_dy_note("111", nickname="有名字")], comments=[], subdir="douyin")
|
||||||
|
await ingest_run(db, run, task, tmp_path)
|
||||||
|
|
||||||
|
run = await _make_run(db, task, started_at=2000)
|
||||||
|
_write_run_dir(tmp_path, [_dy_note("111", nickname="")], comments=[], subdir="douyin")
|
||||||
|
await ingest_run(db, run, task, tmp_path)
|
||||||
|
|
||||||
|
assert (await db.scalar(select(MonitorNote))).creator_name == "有名字"
|
||||||
|
|||||||
Reference in New Issue
Block a user