butubb
61808444ad
**评论能采了。** 之前 comment/list 一直回 200 + 空 body，被读成「这条没评论」—— 而它其实只是被网关挡了。缺的就是 a_bogus 签名，仓库里本来就有 （libs/douyin.js + execjs）。签上之后实测 200 / 9960 字节真评论。 只给评论接口签：作品、详情、博主资料三个不带签名也照常返回，而给它们加签名是 没验证过的改动。签名按需 import —— 那个模块 import 时就把 JS 喂给 execjs， 不该拖进监控层热路径。 **作品导出那几列一直是空的。** 列名写的是裸键 liked_count，而作品行的指标嵌在 metrics / deltas 里，row.get() 永远取到 None —— 导出来的表有「点赞/评论/收藏/ 分享」四列，每一格都没有数。原来的测试只断言了「作品ID」，所以没发现。 顺手补上：导出带上 博主备注/昵称、作品备注、发布时间，时间戳格式化成人能读的 形态（原来是一串 13 位毫秒，Excel 里没法看也没法排序）。
402 lines
15 KiB
Python
402 lines
15 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""抖音 Web 接口客户端 —— 纯逻辑部分（不发网络请求、不连浏览器）。
|
|
|
|
发请求那半边只能在真环境里验（要 CDP 浏览器 + 登录态），所以这里钉住的是那些
|
|
「错了会一路错到入库」的地方：请求头的成套性、cookie 解析、以及产物键名。
|
|
"""
|
|
|
|
from tools.user_hash import anonymize_user_id
|
|
|
|
import asyncio
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from api.monitor import douyin_api
|
|
|
|
|
|
class TestCookieParsing:
|
|
def test_cookie_header_is_normalised(self):
|
|
assert douyin_api._cookie_header(" a=1 ; b = 2 ;; c=3 ") == "a=1; b=2; c=3"
|
|
|
|
def test_cookie_value_lookup(self):
|
|
assert douyin_api._cookie_value("a=1; UIFID=xyz; b=2", "UIFID") == "xyz"
|
|
assert douyin_api._cookie_value("a=1", "UIFID") == ""
|
|
assert douyin_api._cookie_value("", "UIFID") == ""
|
|
|
|
def test_session_detection(self):
|
|
assert douyin_api._has_session("a=1; sessionid=abc") is True
|
|
assert douyin_api._has_session("a=1; sessionid_ss=abc") is False
|
|
assert douyin_api._has_session("") is False
|
|
|
|
|
|
class TestRequestHeaders:
|
|
"""请求头必须**成套**，而且成套地来自同一个浏览器。
|
|
|
|
实测：只有 UA + client hints + Cookie 时，主页接口回 200 但只有 121 字节（空壳）；
|
|
补上 Accept / Accept-Language / Referer 才变成 7074 字节的真数据。
|
|
"""
|
|
|
|
def test_the_full_set_is_sent(self):
|
|
identity = douyin_api.BrowserIdentity(
|
|
cookie="sessionid=s; UIFID=u1",
|
|
user_agent="UA-of-this-browser",
|
|
client_hints={"sec-ch-ua": '"Chrome";v="155"'},
|
|
)
|
|
|
|
headers = identity.headers()
|
|
|
|
assert headers["User-Agent"] == "UA-of-this-browser"
|
|
assert headers["sec-ch-ua"] == '"Chrome";v="155"'
|
|
assert headers["Accept"], "Accept 系列是主页接口能不能返回真数据的必要条件"
|
|
assert headers["Accept-Language"]
|
|
assert headers["Referer"] == "https://www.douyin.com/"
|
|
assert headers["x-tt-argus"] == douyin_api.ARGUS_HEADER_VALUE
|
|
assert headers["uifid"] == "u1"
|
|
assert headers["Cookie"] == "sessionid=s; UIFID=u1"
|
|
|
|
def test_uifid_is_omitted_when_absent(self):
|
|
"""cookie 里没有 uifid 就别带 —— 送个空值反而更像异常请求。"""
|
|
identity = douyin_api.BrowserIdentity(
|
|
cookie="sessionid=s", user_agent="UA", client_hints={}
|
|
)
|
|
|
|
assert "uifid" not in identity.headers()
|
|
|
|
def test_uifid_temp_is_used_as_a_fallback(self):
|
|
identity = douyin_api.BrowserIdentity(
|
|
cookie="sessionid=s; UIFID_TEMP=temp-1", user_agent="UA", client_hints={}
|
|
)
|
|
|
|
assert identity.headers()["uifid"] == "temp-1"
|
|
|
|
|
|
class TestNormalizeAweme:
|
|
def test_keys_match_what_the_store_writes(self):
|
|
"""键名必须和 store/douyin 一模一样，否则 ingest 一条都读不到。"""
|
|
record = douyin_api.normalize_aweme(
|
|
{
|
|
"aweme_id": 7690458980574358513,
|
|
"desc": "中秋哪儿都堵",
|
|
"create_time": 1790574515,
|
|
"author": {"uid": "776719710825195", "nickname": "AA建材王总"},
|
|
"statistics": {
|
|
"digg_count": 3,
|
|
"comment_count": 1,
|
|
"collect_count": 2,
|
|
"share_count": 0,
|
|
},
|
|
"video": {"cover": {"url_list": ["https://img/cover.jpg"]}},
|
|
}
|
|
)
|
|
|
|
assert record["aweme_id"] == "7690458980574358513"
|
|
assert record["title"] == "中秋哪儿都堵"
|
|
assert record["nickname"] == "AA建材王总"
|
|
assert record["cover_url"] == "https://img/cover.jpg"
|
|
assert (
|
|
record["aweme_url"]
|
|
== "https://www.douyin.com/video/7690458980574358513"
|
|
)
|
|
# 与 store 一致：creator_hash 是 uid 的匿名哈希。
|
|
assert record["creator_hash"] == anonymize_user_id("776719710825195")
|
|
# **秒**。adapters 的 time_scale=1000 会把它换成毫秒 —— 这一层不算毫秒。
|
|
assert record["create_time"] == 1790574515
|
|
# 指标按 store 的形态落成字符串，交给 ingest 的 parse_count 解析。
|
|
assert record["liked_count"] == "3"
|
|
assert record["collected_count"] == "2"
|
|
|
|
def test_missing_fields_do_not_crash(self):
|
|
record = douyin_api.normalize_aweme({"aweme_id": "1"})
|
|
|
|
assert record["aweme_id"] == "1"
|
|
assert record["title"] == ""
|
|
assert record["cover_url"] == ""
|
|
assert record["liked_count"] == "0"
|
|
assert record["create_time"] == 0
|
|
|
|
|
|
class TestIdentityFromPages:
|
|
"""身份得从浏览器里问，但**不能被一个卡死的标签页拖住**。
|
|
|
|
实测过：标签页 URL 为空、渲染进程卡死，``page.evaluate`` 永远不返回；而问身份是采集的
|
|
第一步 —— 没超时的话整轮就挂在那儿，run 永远停在「运行中」。
|
|
"""
|
|
|
|
class _Page:
|
|
def __init__(self, url, *, user_agent=None, hang=False):
|
|
self.url = url
|
|
self._user_agent = user_agent
|
|
self._hang = hang
|
|
self.closed = False
|
|
|
|
async def evaluate(self, expression):
|
|
if self._hang:
|
|
await asyncio.sleep(30) # 模拟渲染进程卡死
|
|
if expression.startswith("() => navigator.userAgentData"):
|
|
return {
|
|
"brands": [{"brand": "Chrome", "version": "155"}],
|
|
"mobile": False,
|
|
"platform": "Linux",
|
|
}
|
|
return self._user_agent
|
|
|
|
async def close(self):
|
|
self.closed = True
|
|
|
|
class _Context:
|
|
def __init__(self, pages, temp=None):
|
|
self.pages = pages
|
|
self._temp = temp
|
|
self.made_temp = False
|
|
|
|
async def new_page(self):
|
|
self.made_temp = True
|
|
if self._temp is None:
|
|
raise AssertionError("这个用例不该走到临时页")
|
|
return self._temp
|
|
|
|
def _fast_timeout(self, monkeypatch):
|
|
monkeypatch.setattr(douyin_api, "EVALUATE_TIMEOUT_SECONDS", 0.05)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_a_hanging_page_is_skipped(self, monkeypatch):
|
|
self._fast_timeout(monkeypatch)
|
|
stuck = self._Page("", hang=True)
|
|
good = self._Page("https://example.com/", user_agent="UA-of-good-page")
|
|
|
|
user_agent, hints = await douyin_api._identity_from_pages(
|
|
self._Context([stuck, good])
|
|
)
|
|
|
|
assert user_agent == "UA-of-good-page"
|
|
assert hints["sec-ch-ua"] == '"Chrome";v="155"'
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_a_page_without_a_user_agent_is_skipped(self, monkeypatch):
|
|
self._fast_timeout(monkeypatch)
|
|
blank = self._Page("", user_agent=None)
|
|
good = self._Page("https://example.com/", user_agent="UA-of-good-page")
|
|
|
|
user_agent, _ = await douyin_api._identity_from_pages(
|
|
self._Context([blank, good])
|
|
)
|
|
|
|
assert user_agent == "UA-of-good-page"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_it_opens_a_temporary_page_when_nothing_else_works(self, monkeypatch):
|
|
self._fast_timeout(monkeypatch)
|
|
stuck = self._Page("", hang=True)
|
|
temp = self._Page("about:blank", user_agent="UA-of-temp-page")
|
|
context = self._Context([stuck], temp=temp)
|
|
|
|
user_agent, _ = await douyin_api._identity_from_pages(context)
|
|
|
|
assert context.made_temp is True
|
|
assert user_agent == "UA-of-temp-page"
|
|
assert temp.closed is True, "临时页问完要关掉，别在操作者的浏览器里留垃圾"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_a_douyin_page_is_preferred(self, monkeypatch):
|
|
"""有抖音页面就先问它 —— 它才是我们要模仿的那个身份。"""
|
|
self._fast_timeout(monkeypatch)
|
|
other = self._Page("https://example.com/", user_agent="UA-of-other")
|
|
douyin = self._Page("https://www.douyin.com/explore", user_agent="UA-of-douyin")
|
|
|
|
user_agent, _ = await douyin_api._identity_from_pages(
|
|
self._Context([other, douyin])
|
|
)
|
|
|
|
assert user_agent == "UA-of-douyin"
|
|
|
|
|
|
class TestGet:
|
|
"""`_get` 的失败路径 —— 它们决定了失败会不会被伪装成「这个博主没作品」。"""
|
|
|
|
@staticmethod
|
|
def _client_returning(monkeypatch, status_code: int, text: str):
|
|
class _Response:
|
|
def json(self):
|
|
import json as _json
|
|
|
|
return _json.loads(self.text)
|
|
|
|
response = _Response()
|
|
response.status_code = status_code
|
|
response.text = text
|
|
|
|
class _Client:
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, *exc):
|
|
return False
|
|
|
|
async def get(self, *args, **kwargs):
|
|
return response
|
|
|
|
monkeypatch.setattr(httpx, "AsyncClient", lambda **kwargs: _Client())
|
|
|
|
def _identity(self):
|
|
return douyin_api.BrowserIdentity(
|
|
cookie="sessionid=s", user_agent="UA", client_hints={}
|
|
)
|
|
|
|
def test_an_empty_body_is_an_error_not_an_empty_result(self, monkeypatch):
|
|
"""「200 + 空 body」是网关拒绝请求的典型回应。
|
|
|
|
必须当场报错 —— 放过去的话，它会在下游变成「这个博主没作品」，把一次失败伪装成
|
|
一条正常的空结果。爬虫那条路就是这么栽的，还被翻译成「账号被封」。
|
|
"""
|
|
self._client_returning(monkeypatch, 200, "")
|
|
|
|
with pytest.raises(douyin_api.DouyinApiError) as excinfo:
|
|
asyncio.run(douyin_api._get("/x", {}, self._identity()))
|
|
|
|
assert "空内容" in str(excinfo.value)
|
|
|
|
def test_a_403_carries_the_gateways_own_message(self, monkeypatch):
|
|
"""抖音难得会说原因，把它带出来，别丢。"""
|
|
self._client_returning(
|
|
monkeypatch, 403, "Blocked by ArgusSecurityPlugin Uifid Not Found"
|
|
)
|
|
|
|
with pytest.raises(douyin_api.DouyinApiError) as excinfo:
|
|
asyncio.run(douyin_api._get("/x", {}, self._identity()))
|
|
|
|
assert "403" in str(excinfo.value)
|
|
assert "Uifid Not Found" in str(excinfo.value)
|
|
|
|
def test_a_200_with_data_is_returned_as_is(self, monkeypatch):
|
|
self._client_returning(monkeypatch, 200, '{"user": {"nickname": "x"}}')
|
|
|
|
assert asyncio.run(douyin_api._get("/x", {}, self._identity())) == {
|
|
"user": {"nickname": "x"}
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def fake_signer(monkeypatch):
|
|
"""把真正的 ``a_bogus`` 签名换成假的。
|
|
|
|
真的那个在 import 的瞬间就要把 ``libs/douyin.js`` 喂给 execjs（还得有 node 和正确的
|
|
相对路径），单元测试不该依赖这些。**签名本身是实测过的**：带上它是 200 + 真评论，
|
|
不带是 200 + 空 body。这里只负责钉住「有没有带上、传对了没有」。
|
|
"""
|
|
import sys
|
|
import types
|
|
|
|
module = types.ModuleType("media_platform.douyin.help")
|
|
calls: list = []
|
|
|
|
def get_a_bogus_from_js(url: str, params: str, user_agent: str) -> str:
|
|
calls.append({"url": url, "params": params, "user_agent": user_agent})
|
|
return "FAKE-BOGUS"
|
|
|
|
module.get_a_bogus_from_js = get_a_bogus_from_js
|
|
monkeypatch.setitem(sys.modules, "media_platform.douyin.help", module)
|
|
return calls
|
|
|
|
|
|
@pytest.fixture
|
|
def recording_client(monkeypatch):
|
|
"""记下实际发出去的那次请求。"""
|
|
sent: dict = {}
|
|
|
|
class _Response:
|
|
status_code = 200
|
|
text = '{"comments": []}'
|
|
|
|
def json(self):
|
|
return {"comments": []}
|
|
|
|
class _Client:
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, *exc):
|
|
return False
|
|
|
|
async def get(self, url, **kwargs):
|
|
sent["url"] = url
|
|
sent.update(kwargs)
|
|
return _Response()
|
|
|
|
monkeypatch.setattr(httpx, "AsyncClient", lambda **kwargs: _Client())
|
|
return sent
|
|
|
|
|
|
class TestCommentSigning:
|
|
"""评论接口**必须**带 a_bogus。
|
|
|
|
不带的话网关回 200 + 空 body —— 那在下游会变成「这条作品没有评论」，把一次被挡住的
|
|
请求伪装成一条正常的空结果。和登录失效长得一模一样，查起来能查半天。
|
|
"""
|
|
|
|
def _identity(self):
|
|
return douyin_api.BrowserIdentity(
|
|
cookie="sessionid=s", user_agent="UA", client_hints={}
|
|
)
|
|
|
|
def test_the_signature_is_computed_over_the_unsigned_params(
|
|
self, fake_signer, recording_client
|
|
):
|
|
asyncio.run(
|
|
douyin_api._get(
|
|
douyin_api.COMMENT_PATH,
|
|
{"aweme_id": "123", "count": 20},
|
|
self._identity(),
|
|
signed=True,
|
|
)
|
|
)
|
|
|
|
assert len(fake_signer) == 1
|
|
# 签名算在**不含 a_bogus** 的那串上 —— 把它自己也算进去是循环的。
|
|
assert fake_signer[0]["params"] == "aweme_id=123&count=20"
|
|
assert fake_signer[0]["url"] == douyin_api.COMMENT_PATH
|
|
assert fake_signer[0]["user_agent"] == "UA"
|
|
|
|
def test_the_signature_goes_out_with_the_request(self, fake_signer, recording_client):
|
|
asyncio.run(
|
|
douyin_api._get(
|
|
douyin_api.COMMENT_PATH, {"aweme_id": "123"}, self._identity(), signed=True
|
|
)
|
|
)
|
|
|
|
assert recording_client["params"]["a_bogus"] == "FAKE-BOGUS"
|
|
assert recording_client["params"]["aweme_id"] == "123"
|
|
|
|
def test_unsigned_calls_never_touch_the_signer(self, fake_signer, recording_client):
|
|
"""作品 / 详情 / 博主资料三个接口不带签名也照常返回。
|
|
|
|
给它们加签名是**没验证过的改动** —— 所以这里钉住「不签」，防止有人图省事把
|
|
signed=True 改成全局默认。
|
|
"""
|
|
asyncio.run(
|
|
douyin_api._get(douyin_api.POSTS_PATH, {"sec_user_id": "x"}, self._identity())
|
|
)
|
|
|
|
assert fake_signer == []
|
|
assert "a_bogus" not in recording_client["params"]
|
|
|
|
def test_a_broken_signer_is_reported_as_such(self, monkeypatch, recording_client):
|
|
"""execjs 起不来时要说出是签名失败，而不是让它变成「没评论」。"""
|
|
import sys
|
|
import types
|
|
|
|
module = types.ModuleType("media_platform.douyin.help")
|
|
|
|
def boom(url, params, user_agent):
|
|
raise RuntimeError("node 没装")
|
|
|
|
module.get_a_bogus_from_js = boom
|
|
monkeypatch.setitem(sys.modules, "media_platform.douyin.help", module)
|
|
|
|
with pytest.raises(douyin_api.DouyinApiError, match="a_bogus"):
|
|
asyncio.run(
|
|
douyin_api._get(
|
|
douyin_api.COMMENT_PATH, {"aweme_id": "1"}, self._identity(), signed=True
|
|
)
|
|
)
|