feat(monitor): 组头上加一个跳到博主主页的按钮
`creator_hash` 是单向的,拼不出主页地址 —— 所以这个 id 必须在采集时单独存下来。 抖音的资料接口本来就返回 sec_uid,只是之前没落库。 * `monitor_creator_stat` 加 `creator_id` 列(老数据默认空,下一次采集补上); * 链接在 `adapters.creator_url()` 里拼,**空串是有意义的返回值**:平台不认识、 或者没拿到 id,就返回空,界面据此不画按钮 —— 画一个点开 404 的按钮比不画糟; * 没有作品的博主照样有链接:他恰恰是你想点进去看的那个人。 已知缺口:小红书那条路(爬虫子进程)根本不落创作者资料,所以那边没有链接。
This commit is contained in:
@@ -107,6 +107,17 @@ class PlatformAdapter:
|
|||||||
人能直接点开看的那一个。"""
|
人能直接点开看的那一个。"""
|
||||||
return f"{self.web_base}{self.note_path}/{note_id}"
|
return f"{self.web_base}{self.note_path}/{note_id}"
|
||||||
|
|
||||||
|
def creator_url(self, creator_id: str) -> str:
|
||||||
|
"""博主主页的可点击链接。
|
||||||
|
|
||||||
|
``creator_id`` 是**平台上的那个主页 id**(抖音是 sec_uid),不是 ``creator_hash``
|
||||||
|
—— 哈希是单向的,拼不出地址来。拿不到 id 的地方(小红书那条路根本不落创作者
|
||||||
|
资料)就返回空串,界面上不画按钮,而不是给一个点开 404 的链接。
|
||||||
|
"""
|
||||||
|
if not creator_id:
|
||||||
|
return ""
|
||||||
|
return f"{self.web_base}{self.creator_path}/{creator_id}"
|
||||||
|
|
||||||
def note_field(self, record: Dict[str, Any], name: str) -> Any:
|
def note_field(self, record: Dict[str, Any], name: str) -> Any:
|
||||||
"""按规范名读作品记录里的原始值(没有就是 None)。"""
|
"""按规范名读作品记录里的原始值(没有就是 None)。"""
|
||||||
return record.get(self.note_fields.get(name, name))
|
return record.get(self.note_fields.get(name, name))
|
||||||
@@ -240,6 +251,20 @@ def has_adapter(platform: str) -> bool:
|
|||||||
return platform in ADAPTERS
|
return platform in ADAPTERS
|
||||||
|
|
||||||
|
|
||||||
|
def creator_url(platform: str, creator_id: str) -> str:
|
||||||
|
"""博主主页链接,**空串是有意义的返回值**。
|
||||||
|
|
||||||
|
平台不认识、或者没拿到主页 id,就返回空串 —— 界面据此不画那个按钮。给一个点开
|
||||||
|
404 的链接,比不画按钮糟得多。
|
||||||
|
|
||||||
|
单独开这个模块级函数(而不是让调用方自己 ``adapter(...)`` 包一层 try),是为了让
|
||||||
|
「哪些情况该空着」只有一处判断。
|
||||||
|
"""
|
||||||
|
if not creator_id or not has_adapter(platform):
|
||||||
|
return ""
|
||||||
|
return adapter(platform).creator_url(creator_id)
|
||||||
|
|
||||||
|
|
||||||
def artifact_dir(platform: str) -> str:
|
def artifact_dir(platform: str) -> str:
|
||||||
"""该平台的爬虫会把 jsonl 落在哪个子目录下。
|
"""该平台的爬虫会把 jsonl 落在哪个子目录下。
|
||||||
|
|
||||||
|
|||||||
@@ -463,6 +463,8 @@ async def author_profile(sec_user_id: str, *, cookie: str = "") -> Dict[str, Any
|
|||||||
str(user.get("uid") or user.get("sec_uid") or "")
|
str(user.get("uid") or user.get("sec_uid") or "")
|
||||||
),
|
),
|
||||||
"nickname": user.get("nickname") or "",
|
"nickname": user.get("nickname") or "",
|
||||||
|
# 主页 id —— 单独存一份,用来拼「跳到他主页」的链接。哈希拼不出地址。
|
||||||
|
"creator_id": str(user.get("sec_uid") or ""),
|
||||||
"unique_id": user.get("unique_id") or "",
|
"unique_id": user.get("unique_id") or "",
|
||||||
"fans": _as_int(user.get("follower_count")),
|
"fans": _as_int(user.get("follower_count")),
|
||||||
"total_favorited": _as_int(user.get("total_favorited")),
|
"total_favorited": _as_int(user.get("total_favorited")),
|
||||||
|
|||||||
@@ -638,6 +638,7 @@ async def _ingest_creator_stats(
|
|||||||
run_id=run.id,
|
run_id=run.id,
|
||||||
creator_hash=creator_hash,
|
creator_hash=creator_hash,
|
||||||
nickname=str(record.get("nickname") or "")[:128],
|
nickname=str(record.get("nickname") or "")[:128],
|
||||||
|
creator_id=str(record.get("creator_id") or "")[:128],
|
||||||
fans=parse_count(record.get("fans")),
|
fans=parse_count(record.get("fans")),
|
||||||
total_favorited=parse_count(record.get("total_favorited")),
|
total_favorited=parse_count(record.get("total_favorited")),
|
||||||
works_count=parse_count(record.get("works")),
|
works_count=parse_count(record.get("works")),
|
||||||
|
|||||||
@@ -370,6 +370,10 @@ class MonitorCreatorStat(MonitorBase):
|
|||||||
run_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True)
|
run_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True)
|
||||||
creator_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
creator_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||||
nickname: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
nickname: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||||
|
# 平台上的**主页 id**(抖音是 sec_uid),用来拼博主主页的跳转链接。
|
||||||
|
# creator_hash 是单向的,拼不出地址 —— 所以这个必须单独存。
|
||||||
|
# 拿不到的地方(小红书那条路不落创作者资料)留空,界面就不画那个按钮。
|
||||||
|
creator_id: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||||
|
|
||||||
# 都可能为 None:平台没给就留空,**不要伪造成 0** —— 0 是"掉到零",和"不知道"
|
# 都可能为 None:平台没给就留空,**不要伪造成 0** —— 0 是"掉到零",和"不知道"
|
||||||
# 在趋势图上是完全不同的两回事。
|
# 在趋势图上是完全不同的两回事。
|
||||||
|
|||||||
@@ -756,6 +756,11 @@ async def list_creators(
|
|||||||
"creator_alias": aliases.get(
|
"creator_alias": aliases.get(
|
||||||
(task_platform.get(row_task, ""), creator_hash), ""
|
(task_platform.get(row_task, ""), creator_hash), ""
|
||||||
),
|
),
|
||||||
|
# 主页跳转。拿不到主页 id(小红书不落创作者资料、或者这个博主还没采过
|
||||||
|
# 资料)就是空串,界面不画按钮 —— 而不是给一个点开 404 的链接。
|
||||||
|
"creator_url": adapters.creator_url(
|
||||||
|
task_platform.get(row_task, ""), stat.creator_id if stat else ""
|
||||||
|
),
|
||||||
"note_count": (work_row or {}).get("note_count", 0),
|
"note_count": (work_row or {}).get("note_count", 0),
|
||||||
"creator_fans": stat.fans if stat else None,
|
"creator_fans": stat.fans if stat else None,
|
||||||
"creator_total_favorited": stat.total_favorited if stat else None,
|
"creator_total_favorited": stat.total_favorited if stat else None,
|
||||||
|
|||||||
@@ -139,6 +139,7 @@ async def _add_stat(
|
|||||||
works: int | None = 42,
|
works: int | None = 42,
|
||||||
creator_hash: str = CREATOR_HASH,
|
creator_hash: str = CREATOR_HASH,
|
||||||
captured_at: int = 1,
|
captured_at: int = 1,
|
||||||
|
creator_id: str = "",
|
||||||
) -> None:
|
) -> None:
|
||||||
async with monitor_db.get_session() as session:
|
async with monitor_db.get_session() as session:
|
||||||
session.add(
|
session.add(
|
||||||
@@ -147,6 +148,7 @@ async def _add_stat(
|
|||||||
run_id=run_id,
|
run_id=run_id,
|
||||||
creator_hash=creator_hash,
|
creator_hash=creator_hash,
|
||||||
nickname=NICKNAME,
|
nickname=NICKNAME,
|
||||||
|
creator_id=creator_id,
|
||||||
fans=fans,
|
fans=fans,
|
||||||
total_favorited=total_favorited,
|
total_favorited=total_favorited,
|
||||||
works_count=works,
|
works_count=works,
|
||||||
@@ -320,3 +322,48 @@ class TestCreatorList:
|
|||||||
assert len(creators) == 2
|
assert len(creators) == 2
|
||||||
assert {row["creator_hash"] for row in creators} == {CREATOR_HASH}
|
assert {row["creator_hash"] for row in creators} == {CREATOR_HASH}
|
||||||
assert sum(row["note_count"] for row in creators) == 2
|
assert sum(row["note_count"] for row in creators) == 2
|
||||||
|
|
||||||
|
|
||||||
|
class TestCreatorProfileLink:
|
||||||
|
"""组头上那个「跳到他主页」的按钮。
|
||||||
|
|
||||||
|
它要的是**平台上的主页 id**,不是 ``creator_hash``(哈希单向,拼不出地址)。所以
|
||||||
|
这个 id 是采集资料时单独存下来的;拿不到就返回空串让界面不画按钮。
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_the_link_is_built_from_the_stored_id(self, client):
|
||||||
|
task_id = await _seed_task(platform="dy")
|
||||||
|
await _add_stat(task_id, run_id=1, fans=1, creator_id="MS4wLjABAAAAsomebody")
|
||||||
|
|
||||||
|
creators = await _creators(client, "dy")
|
||||||
|
|
||||||
|
assert creators[0]["creator_url"] == (
|
||||||
|
"https://www.douyin.com/user/MS4wLjABAAAAsomebody"
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_no_id_means_no_link(self, client):
|
||||||
|
"""没采到主页 id 就不给链接 —— 画一个点开 404 的按钮比不画糟。"""
|
||||||
|
task_id = await _seed_task(platform="dy")
|
||||||
|
await _add_stat(task_id, run_id=1, fans=1, creator_id="")
|
||||||
|
|
||||||
|
assert (await _creators(client, "dy"))[0]["creator_url"] == ""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_creator_with_only_works_has_no_link_either(self, client):
|
||||||
|
"""小红书那条路根本不落创作者资料,所以那边永远是空的 —— 这是已知的、可接受的。"""
|
||||||
|
await _seed(platform="xhs")
|
||||||
|
|
||||||
|
assert (await _creators(client, "xhs"))[0]["creator_url"] == ""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_creator_with_no_works_still_gets_the_link(self, client):
|
||||||
|
"""**没有作品的博主照样能跳过去** —— 他恰恰是你想点进去看看发的什么的那个人。"""
|
||||||
|
task_id = await _seed_task(platform="dy")
|
||||||
|
await _add_stat(task_id, run_id=1, fans=1, creator_id="MS4wLjABAAAAquiet")
|
||||||
|
|
||||||
|
creators = await _creators(client, "dy")
|
||||||
|
|
||||||
|
assert creators[0]["note_count"] == 0
|
||||||
|
assert creators[0]["creator_url"].endswith("/user/MS4wLjABAAAAquiet")
|
||||||
|
|||||||
@@ -852,6 +852,7 @@ def _profile(creator_hash: str = "hash", **extra) -> Dict[str, Any]:
|
|||||||
record: Dict[str, Any] = {
|
record: Dict[str, Any] = {
|
||||||
"creator_hash": creator_hash,
|
"creator_hash": creator_hash,
|
||||||
"nickname": "博主",
|
"nickname": "博主",
|
||||||
|
"creator_id": "MS4wLjABAAAAtestsuid",
|
||||||
"unique_id": "abc",
|
"unique_id": "abc",
|
||||||
"fans": 12000,
|
"fans": 12000,
|
||||||
"total_favorited": 83000,
|
"total_favorited": 83000,
|
||||||
@@ -976,3 +977,36 @@ class TestCreatorStatSnapshots:
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert [s.fans for s in stats] == [12000, 12300]
|
assert [s.fans for s in stats] == [12000, 12300]
|
||||||
|
|
||||||
|
|
||||||
|
class TestCreatorIdIsStored:
|
||||||
|
"""主页 id 要单独存下来 —— ``creator_hash`` 是单向的,拼不出主页地址。"""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_the_profile_id_lands_on_the_snapshot(self, db, tmp_path):
|
||||||
|
task = await _make_task(db, platform="dy")
|
||||||
|
run = await _make_run(db, task, started_at=1)
|
||||||
|
_write_run_dir(
|
||||||
|
tmp_path, [_dy_note("1")], comments=[], subdir="douyin", profiles=[_profile()]
|
||||||
|
)
|
||||||
|
await ingest_run(db, run, task, tmp_path)
|
||||||
|
|
||||||
|
stat = await db.scalar(select(MonitorCreatorStat))
|
||||||
|
|
||||||
|
assert stat.creator_id == "MS4wLjABAAAAtestsuid"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_a_missing_id_is_empty_not_a_crash(self, db, tmp_path):
|
||||||
|
"""老数据/别的平台没有这个字段 —— 空串,界面据此不画按钮。"""
|
||||||
|
task = await _make_task(db, platform="dy")
|
||||||
|
run = await _make_run(db, task, started_at=1)
|
||||||
|
_write_run_dir(
|
||||||
|
tmp_path,
|
||||||
|
[_dy_note("1")],
|
||||||
|
comments=[],
|
||||||
|
subdir="douyin",
|
||||||
|
profiles=[_profile(creator_id="")],
|
||||||
|
)
|
||||||
|
await ingest_run(db, run, task, tmp_path)
|
||||||
|
|
||||||
|
assert (await db.scalar(select(MonitorCreatorStat))).creator_id == ""
|
||||||
|
|||||||
@@ -389,3 +389,30 @@ class TestLegacyKeyMigration:
|
|||||||
assert "xhs_cookie" not in rows
|
assert "xhs_cookie" not in rows
|
||||||
|
|
||||||
await monitor_db.dispose_engine()
|
await monitor_db.dispose_engine()
|
||||||
|
|
||||||
|
|
||||||
|
class TestCreatorUrl:
|
||||||
|
"""博主主页链接。
|
||||||
|
|
||||||
|
注意它要的是**平台上的主页 id**(抖音是 sec_uid),不是 ``creator_hash`` —— 哈希是
|
||||||
|
单向的,拼不出地址。所以这条链路上少任何一环(采集没存 id、平台上没有这个博主、
|
||||||
|
平台不认识)都必须返回空串,让界面干脆不画按钮,而不是给一个点开 404 的链接。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_douyin(self):
|
||||||
|
assert (
|
||||||
|
adapters.creator_url("dy", "MS4wLjABAAAAabc-123")
|
||||||
|
== "https://www.douyin.com/user/MS4wLjABAAAAabc-123"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_xiaohongshu(self):
|
||||||
|
assert (
|
||||||
|
adapters.creator_url("xhs", "5f8a1b2c3d4e5f6a7b8c9d0e")
|
||||||
|
== "https://www.xiaohongshu.com/user/profile/5f8a1b2c3d4e5f6a7b8c9d0e"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_no_id_is_empty_not_a_broken_link(self):
|
||||||
|
assert adapters.creator_url("dy", "") == ""
|
||||||
|
|
||||||
|
def test_an_unknown_platform_is_empty_not_a_crash(self):
|
||||||
|
assert adapters.creator_url("nope", "abc") == ""
|
||||||
|
|||||||
@@ -172,6 +172,8 @@ export function NotesTable({ taskId, onlyNew }: NotesTableProps) {
|
|||||||
key: string
|
key: string
|
||||||
name: string
|
name: string
|
||||||
alias: string
|
alias: string
|
||||||
|
/** 主页链接。空串就是不画那个按钮(见 MonitorCreator.creator_url)。 */
|
||||||
|
url: string
|
||||||
/** 服务端说的这条博主名下有多少作品 —— 列表被 `onlyNew` 滤过时和 `notes.length` 不等。 */
|
/** 服务端说的这条博主名下有多少作品 —— 列表被 `onlyNew` 滤过时和 `notes.length` 不等。 */
|
||||||
totalNotes: number
|
totalNotes: number
|
||||||
stats: MonitorCreator | null
|
stats: MonitorCreator | null
|
||||||
@@ -194,12 +196,14 @@ export function NotesTable({ taskId, onlyNew }: NotesTableProps) {
|
|||||||
existing.name ||= creator.creator_name
|
existing.name ||= creator.creator_name
|
||||||
// 备注按 (platform, creator_hash) 存,所以跨任务就是同一条,取到即可。
|
// 备注按 (platform, creator_hash) 存,所以跨任务就是同一条,取到即可。
|
||||||
existing.alias ||= creator.creator_alias
|
existing.alias ||= creator.creator_alias
|
||||||
|
existing.url ||= creator.creator_url
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
byCreator.set(key, {
|
byCreator.set(key, {
|
||||||
key,
|
key,
|
||||||
name: creator.creator_name || '',
|
name: creator.creator_name || '',
|
||||||
alias: creator.creator_alias || '',
|
alias: creator.creator_alias || '',
|
||||||
|
url: creator.creator_url || '',
|
||||||
totalNotes: creator.note_count,
|
totalNotes: creator.note_count,
|
||||||
stats: creator,
|
stats: creator,
|
||||||
notes: [],
|
notes: [],
|
||||||
@@ -216,6 +220,7 @@ export function NotesTable({ taskId, onlyNew }: NotesTableProps) {
|
|||||||
key,
|
key,
|
||||||
name: note.creator_name || '',
|
name: note.creator_name || '',
|
||||||
alias: note.creator_alias || '',
|
alias: note.creator_alias || '',
|
||||||
|
url: '',
|
||||||
totalNotes: 0,
|
totalNotes: 0,
|
||||||
stats: null,
|
stats: null,
|
||||||
notes: [],
|
notes: [],
|
||||||
@@ -320,6 +325,20 @@ export function NotesTable({ taskId, onlyNew }: NotesTableProps) {
|
|||||||
{group.alias && group.name && (
|
{group.alias && group.name && (
|
||||||
<span className="text-cyber-text-muted truncate">{group.name}</span>
|
<span className="text-cyber-text-muted truncate">{group.name}</span>
|
||||||
)}
|
)}
|
||||||
|
{/* 跳到他主页。链接是采集时从资料接口带回来的(哈希拼不出地址),
|
||||||
|
拿不到就整块不画 —— 一个点开 404 的按钮比没有按钮糟。 */}
|
||||||
|
{group.url && (
|
||||||
|
<a
|
||||||
|
href={group.url}
|
||||||
|
target="_blank"
|
||||||
|
rel="noopener noreferrer"
|
||||||
|
onClick={(event) => event.stopPropagation()}
|
||||||
|
title="打开博主主页"
|
||||||
|
className="text-cyber-text-muted hover:text-cyber-neon-cyan flex-shrink-0"
|
||||||
|
>
|
||||||
|
<ExternalLink className="w-3 h-3" />
|
||||||
|
</a>
|
||||||
|
)}
|
||||||
<span className="text-cyber-text-muted flex-shrink-0">
|
<span className="text-cyber-text-muted flex-shrink-0">
|
||||||
{/* 被 onlyNew 滤过时,把「显示了几篇 / 一共几篇」都说出来,
|
{/* 被 onlyNew 滤过时,把「显示了几篇 / 一共几篇」都说出来,
|
||||||
否则「1 篇」会让人以为这个号只发过一条。 */}
|
否则「1 篇」会让人以为这个号只发过一条。 */}
|
||||||
|
|||||||
@@ -151,6 +151,14 @@ export interface MonitorCreator {
|
|||||||
creator_hash: string
|
creator_hash: string
|
||||||
creator_name: string
|
creator_name: string
|
||||||
creator_alias: string
|
creator_alias: string
|
||||||
|
/**
|
||||||
|
* 博主主页链接,用来一键跳过去。
|
||||||
|
*
|
||||||
|
* **空串是常态,不是异常**:`creator_hash` 是单向的,拼不出地址,所以这个得靠采集时
|
||||||
|
* 单独存下来的主页 id(抖音是 sec_uid)。小红书那条路不落创作者资料,因此那边永远是
|
||||||
|
* 空的 —— 空的时候界面不画按钮,而不是给一个点开 404 的链接。
|
||||||
|
*/
|
||||||
|
creator_url: string
|
||||||
note_count: number
|
note_count: number
|
||||||
creator_fans: number | null
|
creator_fans: number | null
|
||||||
creator_total_favorited: number | null
|
creator_total_favorited: number | null
|
||||||
|
|||||||
Reference in New Issue
Block a user