butubb/MediaCrawler
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

i18n: translate all Chinese comments, docstrings, and logger messages to English

Browse Source 
Comprehensive translation of Chinese text to English across the entire codebase:

- api/: FastAPI server documentation and logger messages
- cache/: Cache abstraction layer comments and docstrings
- database/: Database models and MongoDB store documentation
- media_platform/: All platform crawlers (Bilibili, Douyin, Kuaishou, Tieba, Weibo, Xiaohongshu, Zhihu)
- model/: Data model documentation
- proxy/: Proxy pool and provider documentation
- store/: Data storage layer comments
- tools/: Utility functions and browser automation
- test/: Test file documentation

Preserved: Chinese disclaimer header (lines 10-18) for legal compliance

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <[email protected]>
This commit is contained in:
程序员阿江(Relakkes)
2025-12-26 23:27:19 +08:00
co-authored by Claude Opus 4.5
parent 1544d13dd5
commit 157ddfb21b
93 changed files with 1971 additions and 1955 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media_platform/bilibili/client.py
+54 -54
View File
@@ -20,7 +20,7 @@
# -*- coding: utf-8 -*-
# @Author : [email protected]
# @Time : 2023/12/2 18:44
# @Desc : bilibili 请求客户端
# @Desc : bilibili request client
import asyncio
import json
import random
@@ -47,7 +47,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
def __init__(
self,
timeout=60, # 若开启爬取媒体选项，b 站的长视频需要更久的超时时间
timeout=60, # For media crawling, Bilibili long videos need a longer timeout
proxy=None,
*,
headers: Dict[str, str],
@@ -61,11 +61,11 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
self._host = "https://api.bilibili.com"
self.playwright_page = playwright_page
self.cookie_dict = cookie_dict
# 初始化代理池（来自 ProxyRefreshMixin）
# Initialize proxy pool (from ProxyRefreshMixin)
self.init_proxy_pool(proxy_ip_pool)
async def request(self, method, url, **kwargs) -> Any:
# 每次请求前检测代理是否过期
# Check if proxy has expired before each request
await self._refresh_proxy_if_expired()
async with httpx.AsyncClient(proxy=self.proxy) as client:
@@ -82,8 +82,8 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
async def pre_request_data(self, req_data: Dict) -> Dict:
"""
发送请求进行请求参数签名
需要从 localStorage 拿 wbi_img_urls 这参数，值如下：
Send request to sign request parameters
Need to get wbi_img_urls parameter from localStorage, value as follows:
https://i0.hdslb.com/bfs/wbi/7cd084941338484aae1ad9425b84077c.png-https://i0.hdslb.com/bfs/wbi/4932caff0ff746eab6f01bf08b70ac45.png
:param req_data:
:return:
@@ -95,7 +95,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
async def get_wbi_keys(self) -> Tuple[str, str]:
"""
获取最新的 img_key 和 sub_key
Get the latest img_key and sub_key
:return:
"""
local_storage = await self.playwright_page.evaluate("() => window.localStorage")
@@ -160,12 +160,12 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
) -> Dict:
"""
KuaiShou web search api
:param keyword: 搜索关键词
:param page: 分页参数具体第几页
:param page_size: 每一页参数的数量
:param order: 搜索结果排序，默认位综合排序
:param pubtime_begin_s: 发布时间开始时间戳
:param pubtime_end_s: 发布时间结束时间戳
:param keyword: Search keyword
:param page: Page number for pagination
:param page_size: Number of items per page
:param order: Sort order for search results, default is comprehensive sorting
:param pubtime_begin_s: Publish time start timestamp
:param pubtime_end_s: Publish time end timestamp
:return:
"""
uri = "/x/web-interface/wbi/search/type"
@@ -182,13 +182,13 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
async def get_video_info(self, aid: Union[int, None] = None, bvid: Union[str, None] = None) -> Dict:
"""
Bilibli web video detail api, aid 和 bvid任选一个参数
:param aid: 稿件avid
:param bvid: 稿件bvid
Bilibli web video detail api, choose one parameter between aid and bvid
:param aid: Video aid
:param bvid: Video bvid
:return:
"""
if not aid and not bvid:
raise ValueError("请提供 aid 或 bvid 中的至少一个参数")
raise ValueError("Please provide at least one parameter: aid or bvid")
uri = "/x/web-interface/view/detail"
params = dict()
@@ -201,12 +201,12 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
async def get_video_play_url(self, aid: int, cid: int) -> Dict:
"""
Bilibli web video play url api
:param aid: 稿件avid
:param aid: Video aid
:param cid: cid
:return:
"""
if not aid or not cid or aid <= 0 or cid <= 0:
raise ValueError("aid 和 cid 必须存在")
raise ValueError("aid and cid must exist")
uri = "/x/player/wbi/playurl"
qn_value = getattr(config, "BILI_QN", 80)
params = {
@@ -233,7 +233,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
)
return None
except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
utils.logger.error(f"[BilibiliClient.get_video_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # 保留原始异常类型名称，以便开发者调试
utils.logger.error(f"[BilibiliClient.get_video_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # Keep original exception type name for developer debugging
return None
async def get_video_comments(
@@ -243,9 +243,9 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
next: int = 0,
) -> Dict:
"""get video comments
:param video_id: 视频 ID
:param order_mode: 排序方式
:param next: 评论页选择
:param video_id: Video ID
:param order_mode: Sort order
:param next: Comment page selection
:return:
"""
uri = "/x/v2/reply/wbi/main"
@@ -266,7 +266,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
:param crawl_interval:
:param is_fetch_sub_comments:
:param callback:
max_count: 一次笔记爬取的最大评论数量
max_count: Maximum number of comments to crawl per note
:return:
"""
@@ -299,7 +299,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
comment_list: List[Dict] = comments_res.get("replies", [])
# 检查 is_end 和 next 是否存在
# Check if is_end and next exist
if "is_end" not in cursor_info or "next" not in cursor_info:
utils.logger.warning(f"[BilibiliClient.get_video_all_comments] 'is_end' or 'next' not in cursor for video_id: {video_id}. Assuming end of comments.")
is_end = True
@@ -317,7 +317,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
{await self.get_video_all_level_two_comments(video_id, comment_id, CommentOrderType.DEFAULT, 10, crawl_interval, callback)}
if len(result) + len(comment_list) > max_count:
comment_list = comment_list[:max_count - len(result)]
if callback: # 如果有回调函数，就执行回调函数
if callback: # If there is a callback function, execute it
await callback(video_id, comment_list)
await asyncio.sleep(crawl_interval)
if not is_fetch_sub_comments:
@@ -336,10 +336,10 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
) -> Dict:
"""
get video all level two comments for a level one comment
:param video_id: 视频 ID
:param level_one_comment_id: 一级评论 ID
:param video_id: Video ID
:param level_one_comment_id: Level one comment ID
:param order_mode:
:param ps: 一页评论数
:param ps: Number of comments per page
:param crawl_interval:
:param callback:
:return:
@@ -349,7 +349,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
while True:
result = await self.get_video_level_two_comments(video_id, level_one_comment_id, pn, ps, order_mode)
comment_list: List[Dict] = result.get("replies", [])
if callback: # 如果有回调函数，就执行回调函数
if callback: # If there is a callback function, execute it
await callback(video_id, comment_list)
await asyncio.sleep(crawl_interval)
if (int(result["page"]["count"]) <= pn * ps):
@@ -366,9 +366,9 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
order_mode: CommentOrderType,
) -> Dict:
"""get video level two comments
:param video_id: 视频 ID
:param level_one_comment_id: 一级评论 ID
:param order_mode: 排序方式
:param video_id: Video ID
:param level_one_comment_id: Level one comment ID
:param order_mode: Sort order
:return:
"""
@@ -386,10 +386,10 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
async def get_creator_videos(self, creator_id: str, pn: int, ps: int = 30, order_mode: SearchOrderType = SearchOrderType.LAST_PUBLISH) -> Dict:
"""get all videos for a creator
:param creator_id: 创作者 ID
:param pn: 页数
:param ps: 一页视频数
:param order_mode: 排序方式
:param creator_id: Creator ID
:param pn: Page number
:param ps: Number of videos per page
:param order_mode: Sort order
:return:
"""
@@ -405,7 +405,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
async def get_creator_info(self, creator_id: int) -> Dict:
"""
get creator info
:param creator_id: 作者 ID
:param creator_id: Creator ID
"""
uri = "/x/space/wbi/acc/info"
post_data = {
@@ -421,9 +421,9 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
) -> Dict:
"""
get creator fans
:param creator_id: 创作者 ID
:param pn: 开始页数
:param ps: 每页数量
:param creator_id: Creator ID
:param pn: Start page number
:param ps: Number of items per page
:return:
"""
uri = "/x/relation/fans"
@@ -443,9 +443,9 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
) -> Dict:
"""
get creator followings
:param creator_id: 创作者 ID
:param pn: 开始页数
:param ps: 每页数量
:param creator_id: Creator ID
:param pn: Start page number
:param ps: Number of items per page
:return:
"""
uri = "/x/relation/followings"
@@ -460,8 +460,8 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
async def get_creator_dynamics(self, creator_id: int, offset: str = ""):
"""
get creator comments
:param creator_id: 创作者 ID
:param offset: 发送请求所需参数
:param creator_id: Creator ID
:param offset: Parameter required for sending request
:return:
"""
uri = "/x/polymer/web-dynamic/v1/feed/space"
@@ -485,9 +485,9 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
:param creator_info:
:param crawl_interval:
:param callback:
:param max_count: 一个up主爬取的最大粉丝数量
:param max_count: Maximum number of fans to crawl for a creator
:return: up主粉丝数列表
:return: List of creator fans
"""
creator_id = creator_info["id"]
result = []
@@ -499,7 +499,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
pn += 1
if len(result) + len(fans_list) > max_count:
fans_list = fans_list[:max_count - len(result)]
if callback: # 如果有回调函数，就执行回调函数
if callback: # If there is a callback function, execute it
await callback(creator_info, fans_list)
await asyncio.sleep(crawl_interval)
if not fans_list:
@@ -519,9 +519,9 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
:param creator_info:
:param crawl_interval:
:param callback:
:param max_count: 一个up主爬取的最大关注者数量
:param max_count: Maximum number of followings to crawl for a creator
:return: up主关注者列表
:return: List of creator followings
"""
creator_id = creator_info["id"]
result = []
@@ -533,7 +533,7 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
pn += 1
if len(result) + len(followings_list) > max_count:
followings_list = followings_list[:max_count - len(result)]
if callback: # 如果有回调函数，就执行回调函数
if callback: # If there is a callback function, execute it
await callback(creator_info, followings_list)
await asyncio.sleep(crawl_interval)
if not followings_list:
@@ -553,9 +553,9 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
:param creator_info:
:param crawl_interval:
:param callback:
:param max_count: 一个up主爬取的最大动态数量
:param max_count: Maximum number of dynamics to crawl for a creator
:return: up主关注者列表
:return: List of creator dynamics
"""
creator_id = creator_info["id"]
result = []
media_platform/bilibili/core.py
+30 -30
View File
@@ -20,7 +20,7 @@
# -*- coding: utf-8 -*-
# @Author : [email protected]
# @Time : 2023/12/2 18:44
# @Desc : B站爬虫
# @Desc : Bilibili Crawler
import asyncio
import os
@@ -64,7 +64,7 @@ class BilibiliCrawler(AbstractCrawler):
self.index_url = "https://www.bilibili.com"
self.user_agent = utils.get_user_agent()
self.cdp_manager = None
self.ip_proxy_pool = None # 代理IP池，用于代理自动刷新
self.ip_proxy_pool = None # Proxy IP pool for automatic proxy refresh
async def start(self):
playwright_proxy_format, httpx_proxy_format = None, None
@@ -74,9 +74,9 @@ class BilibiliCrawler(AbstractCrawler):
playwright_proxy_format, httpx_proxy_format = utils.format_proxy_info(ip_proxy_info)
async with async_playwright() as playwright:
# 根据配置选择启动模式
# Choose launch mode based on configuration
if config.ENABLE_CDP_MODE:
utils.logger.info("[BilibiliCrawler] 使用CDP模式启动浏览器")
utils.logger.info("[BilibiliCrawler] Launching browser using CDP mode")
self.browser_context = await self.launch_browser_with_cdp(
playwright,
playwright_proxy_format,
@@ -84,7 +84,7 @@ class BilibiliCrawler(AbstractCrawler):
headless=config.CDP_HEADLESS,
)
else:
utils.logger.info("[BilibiliCrawler] 使用标准模式启动浏览器")
utils.logger.info("[BilibiliCrawler] Launching browser using standard mode")
# Launch a browser context.
chromium = playwright.chromium
self.browser_context = await self.launch_browser(chromium, None, self.user_agent, headless=config.HEADLESS)
@@ -149,31 +149,31 @@ class BilibiliCrawler(AbstractCrawler):
end: str = config.END_DAY,
) -> Tuple[str, str]:
"""
获取 bilibili 作品发布日期起始时间戳 pubtime_begin_s 与发布日期结束时间戳 pubtime_end_s
Get bilibili publish start timestamp pubtime_begin_s and publish end timestamp pubtime_end_s
---
:param start: 发布日期起始时间，YYYY-MM-DD
:param end: 发布日期结束时间，YYYY-MM-DD
:param start: Publish date start time, YYYY-MM-DD
:param end: Publish date end time, YYYY-MM-DD
Note
---
- 搜索的时间范围为 start 至 end，包含 start 和 end
- 若要搜索同一天的内容，为了包含 start 当天的搜索内容，则 pubtime_end_s 的值应该为 pubtime_begin_s 的值加上一天再减去一秒，即 start 当天的最后一秒
- 如仅搜索 2024-01-05 的内容，pubtime_begin_s = 1704384000，pubtime_end_s = 1704470399
转换为可读的 datetime 对象：pubtime_begin_s = datetime.datetime(2024, 1, 5, 0, 0)，pubtime_end_s = datetime.datetime(2024, 1, 5, 23, 59, 59)
- 若要搜索 start 至 end 的内容，为了包含 end 当天的搜索内容，则 pubtime_end_s 的值应该为 pubtime_end_s 的值加上一天再减去一秒，即 end 当天的最后一秒
- 如搜索 2024-01-05 - 2024-01-06 的内容，pubtime_begin_s = 1704384000，pubtime_end_s = 1704556799
转换为可读的 datetime 对象：pubtime_begin_s = datetime.datetime(2024, 1, 5, 0, 0)，pubtime_end_s = datetime.datetime(2024, 1, 6, 23, 59, 59)
- Search time range is from start to end, including both start and end
- To search content from the same day, to include search content from that day, pubtime_end_s should be pubtime_begin_s plus one day minus one second, i.e., the last second of start day
- For example, searching only 2024-01-05 content, pubtime_begin_s = 1704384000, pubtime_end_s = 1704470399
Converted to readable datetime objects: pubtime_begin_s = datetime.datetime(2024, 1, 5, 0, 0), pubtime_end_s = datetime.datetime(2024, 1, 5, 23, 59, 59)
- To search content from start to end, to include search content from end day, pubtime_end_s should be pubtime_end_s plus one day minus one second, i.e., the last second of end day
- For example, searching 2024-01-05 - 2024-01-06 content, pubtime_begin_s = 1704384000, pubtime_end_s = 1704556799
Converted to readable datetime objects: pubtime_begin_s = datetime.datetime(2024, 1, 5, 0, 0), pubtime_end_s = datetime.datetime(2024, 1, 6, 23, 59, 59)
"""
# 转换 start 与 end 为 datetime 对象
# Convert start and end to datetime objects
start_day: datetime = datetime.strptime(start, "%Y-%m-%d")
end_day: datetime = datetime.strptime(end, "%Y-%m-%d")
if start_day > end_day:
raise ValueError("Wrong time range, please check your start and end argument, to ensure that the start cannot exceed end")
elif start_day == end_day: # 搜索同一天的内容
end_day = (start_day + timedelta(days=1) - timedelta(seconds=1)) # 则将 end_day 设置为 start_day + 1 day - 1 second
else: # 搜索 start 至 end
end_day = (end_day + timedelta(days=1) - timedelta(seconds=1)) # 则将 end_day 设置为 end_day + 1 day - 1 second
# 将其重新转换为时间戳
elif start_day == end_day: # Searching content from the same day
end_day = (start_day + timedelta(days=1) - timedelta(seconds=1)) # Set end_day to start_day + 1 day - 1 second
else: # Searching from start to end
end_day = (end_day + timedelta(days=1) - timedelta(seconds=1)) # Set end_day to end_day + 1 day - 1 second
# Convert back to timestamps
return str(int(start_day.timestamp())), str(int(end_day.timestamp()))
async def search_by_keywords(self):
@@ -203,8 +203,8 @@ class BilibiliCrawler(AbstractCrawler):
page=page,
page_size=bili_limit_count,
order=SearchOrderType.DEFAULT,
pubtime_begin_s=0, # 作品发布日期起始时间戳
pubtime_end_s=0, # 作品发布日期结束日期时间戳
pubtime_begin_s=0, # Publish date start timestamp
pubtime_end_s=0, # Publish date end timestamp
)
video_list: List[Dict] = videos_res.get("result")
@@ -508,7 +508,7 @@ class BilibiliCrawler(AbstractCrawler):
"height": 1080
},
user_agent=user_agent,
channel="chrome", # 使用系统的Chrome稳定版
channel="chrome", # Use system's stable Chrome version
)
return browser_context
else:
@@ -525,7 +525,7 @@ class BilibiliCrawler(AbstractCrawler):
headless: bool = True,
) -> BrowserContext:
"""
使用CDP模式启动浏览器
Launch browser using CDP mode
"""
try:
self.cdp_manager = CDPBrowserManager()
@@ -536,22 +536,22 @@ class BilibiliCrawler(AbstractCrawler):
headless=headless,
)
# 显示浏览器信息
# Display browser information
browser_info = await self.cdp_manager.get_browser_info()
utils.logger.info(f"[BilibiliCrawler] CDP浏览器信息: {browser_info}")
utils.logger.info(f"[BilibiliCrawler] CDP browser info: {browser_info}")
return browser_context
except Exception as e:
utils.logger.error(f"[BilibiliCrawler] CDP模式启动失败，回退到标准模式: {e}")
# 回退到标准模式
utils.logger.error(f"[BilibiliCrawler] CDP mode launch failed, fallback to standard mode: {e}")
# Fallback to standard mode
chromium = playwright.chromium
return await self.launch_browser(chromium, playwright_proxy, user_agent, headless)
async def close(self):
"""Close browser context"""
try:
# 如果使用CDP模式，需要特殊处理
# If using CDP mode, special handling is required
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
media_platform/bilibili/field.py
+8 -8
View File
@@ -27,28 +27,28 @@ from enum import Enum
class SearchOrderType(Enum):
# 综合排序
# Comprehensive sorting
DEFAULT = ""
# 最多点击
# Most clicks
MOST_CLICK = "click"
# 最新发布
# Latest published
LAST_PUBLISH = "pubdate"
# 最多弹幕
# Most danmu (comments)
MOST_DANMU = "dm"
# 最多收藏
# Most bookmarks
MOST_MARK = "stow"
class CommentOrderType(Enum):
# 仅按热度
# By popularity only
DEFAULT = 0
# 按热度+按时间
# By popularity + time
MIXED = 1
# 按时间
# By time
TIME = 2
media_platform/bilibili/help.py
+27 -27
View File
@@ -21,8 +21,8 @@
# -*- coding: utf-8 -*-
# @Author : [email protected]
# @Time : 2023/12/2 23:26
# @Desc : bilibili 请求参数签名
# 逆向实现参考：https://socialsisteryi.github.io/bilibili-API-collect/docs/misc/sign/wbi.html#wbi%E7%AD%BE%E5%90%8D%E7%AE%97%E6%B3%95
# @Desc : bilibili request parameter signing
# Reverse engineering implementation reference: https://socialsisteryi.github.io/bilibili-API-collect/docs/misc/sign/wbi.html#wbi%E7%AD%BE%E5%90%8D%E7%AE%97%E6%B3%95
import re
import urllib.parse
from hashlib import md5
@@ -45,7 +45,7 @@ class BilibiliSign:
def get_salt(self) -> str:
"""
获取加盐的 key
Get the salted key
:return:
"""
salt = ""
@@ -56,8 +56,8 @@ class BilibiliSign:
def sign(self, req_data: Dict) -> Dict:
"""
请求参数中加上当前时间戳对请求参数中的key进行字典序排序
再将请求参数进行 url 编码集合 salt 进行 md5 就可以生成w_rid参数了
Add current timestamp to request parameters, sort keys in dictionary order,
then URL encode the parameters and combine with salt to generate md5 for w_rid parameter
:param req_data:
:return:
"""
@@ -65,35 +65,35 @@ class BilibiliSign:
req_data.update({"wts": current_ts})
req_data = dict(sorted(req_data.items()))
req_data = {
# 过滤 value 中的 "!'()*" 字符
# Filter "!'()*" characters from values
k: ''.join(filter(lambda ch: ch not in "!'()*", str(v)))
for k, v
in req_data.items()
}
query = urllib.parse.urlencode(req_data)
salt = self.get_salt()
wbi_sign = md5((query + salt).encode()).hexdigest() # 计算 w_rid
wbi_sign = md5((query + salt).encode()).hexdigest() # Calculate w_rid
req_data['w_rid'] = wbi_sign
return req_data
def parse_video_info_from_url(url: str) -> VideoUrlInfo:
"""
从B站视频URL中解析出视频ID
Parse video ID from Bilibili video URL
Args:
url: B站视频链接
url: Bilibili video link
- https://www.bilibili.com/video/BV1dwuKzmE26/?spm_id_from=333.1387.homepage.video_card.click
- https://www.bilibili.com/video/BV1d54y1g7db
- BV1d54y1g7db (直接传入BV号)
- BV1d54y1g7db (directly pass BV number)
Returns:
VideoUrlInfo: 包含视频ID的对象
VideoUrlInfo: Object containing video ID
"""
# 如果传入的已经是BV号,直接返回
# If the input is already a BV number, return directly
if url.startswith("BV"):
return VideoUrlInfo(video_id=url)
# 使用正则表达式提取BV号
# 匹配 /video/BV... 或 /video/av... 格式
# Use regex to extract BV number
# Match /video/BV... or /video/av... format
bv_pattern = r'/video/(BV[a-zA-Z0-9]+)'
match = re.search(bv_pattern, url)
@@ -101,26 +101,26 @@ def parse_video_info_from_url(url: str) -> VideoUrlInfo:
video_id = match.group(1)
return VideoUrlInfo(video_id=video_id)
raise ValueError(f"无法从URL中解析出视频ID: {url}")
raise ValueError(f"Unable to parse video ID from URL: {url}")
def parse_creator_info_from_url(url: str) -> CreatorUrlInfo:
"""
从B站创作者空间URL中解析出创作者ID
Parse creator ID from Bilibili creator space URL
Args:
url: B站创作者空间链接
url: Bilibili creator space link
- https://space.bilibili.com/434377496?spm_id_from=333.1007.0.0
- https://space.bilibili.com/20813884
- 434377496 (直接传入UID)
- 434377496 (directly pass UID)
Returns:
CreatorUrlInfo: 包含创作者ID的对象
CreatorUrlInfo: Object containing creator ID
"""
# 如果传入的已经是纯数字ID,直接返回
# If the input is already a numeric ID, return directly
if url.isdigit():
return CreatorUrlInfo(creator_id=url)
# 使用正则表达式提取UID
# 匹配 /space.bilibili.com/数字 格式
# Use regex to extract UID
# Match /space.bilibili.com/number format
uid_pattern = r'space\.bilibili\.com/(\d+)'
match = re.search(uid_pattern, url)
@@ -128,20 +128,20 @@ def parse_creator_info_from_url(url: str) -> CreatorUrlInfo:
creator_id = match.group(1)
return CreatorUrlInfo(creator_id=creator_id)
raise ValueError(f"无法从URL中解析出创作者ID: {url}")
raise ValueError(f"Unable to parse creator ID from URL: {url}")
if __name__ == '__main__':
# 测试视频URL解析
# Test video URL parsing
video_url1 = "https://www.bilibili.com/video/BV1dwuKzmE26/?spm_id_from=333.1387.homepage.video_card.click"
video_url2 = "BV1d54y1g7db"
print("视频URL解析测试:")
print("Video URL parsing test:")
print(f"URL1: {video_url1} -> {parse_video_info_from_url(video_url1)}")
print(f"URL2: {video_url2} -> {parse_video_info_from_url(video_url2)}")
# 测试创作者URL解析
# Test creator URL parsing
creator_url1 = "https://space.bilibili.com/434377496?spm_id_from=333.1007.0.0"
creator_url2 = "20813884"
print("\n创作者URL解析测试:")
print("\nCreator URL parsing test:")
print(f"URL1: {creator_url1} -> {parse_creator_info_from_url(creator_url1)}")
print(f"URL2: {creator_url2} -> {parse_creator_info_from_url(creator_url2)}")
media_platform/bilibili/login.py
+1 -1
View File
@@ -21,7 +21,7 @@
# -*- coding: utf-8 -*-
# @Author : [email protected]
# @Time : 2023/12/2 18:44
# @Desc : bilibli登录实现类
# @Desc : bilibili login implementation class
import asyncio
import functools
media_platform/douyin/field.py
+11 -11
View File
@@ -23,21 +23,21 @@ from enum import Enum
class SearchChannelType(Enum):
"""search channel type"""
GENERAL = "aweme_general" # 综合
VIDEO = "aweme_video_web" # 视频
USER = "aweme_user_web" # 用户
LIVE = "aweme_live" # 直播
GENERAL = "aweme_general" # General
VIDEO = "aweme_video_web" # Video
USER = "aweme_user_web" # User
LIVE = "aweme_live" # Live
class SearchSortType(Enum):
"""search sort type"""
GENERAL = 0 # 综合排序
MOST_LIKE = 1 # 最多点赞
LATEST = 2 # 最新发布
GENERAL = 0 # Comprehensive sorting
MOST_LIKE = 1 # Most likes
LATEST = 2 # Latest published
class PublishTimeType(Enum):
"""publish time type"""
UNLIMITED = 0 # 不限
ONE_DAY = 1 # 一天内
ONE_WEEK = 7 # 一周内
SIX_MONTH = 180 # 半年内
UNLIMITED = 0 # Unlimited
ONE_DAY = 1 # Within one day
ONE_WEEK = 7 # Within one week
SIX_MONTH = 180 # Within six months
media_platform/douyin/help.py
+37 -37
View File
@@ -22,7 +22,7 @@
# @Author : [email protected]
# @Name : 程序员阿江-Relakkes
# @Time : 2024/6/10 02:24
# @Desc : 获取 a_bogus 参数, 学习交流使用，请勿用作商业用途，侵权联系作者删除
# @Desc : Get a_bogus parameter, for learning and communication only, do not use for commercial purposes, contact author to delete if infringement
import random
import re
@@ -38,7 +38,7 @@ douyin_sign_obj = execjs.compile(open('libs/douyin.js', encoding='utf-8-sig').re
def get_web_id():
"""
生成随机的webid
Generate random webid
Returns:
"""
@@ -60,13 +60,13 @@ def get_web_id():
async def get_a_bogus(url: str, params: str, post_data: dict, user_agent: str, page: Page = None):
"""
获取 a_bogus 参数, 目前不支持post请求类型的签名
Get a_bogus parameter, currently does not support POST request type signature
"""
return get_a_bogus_from_js(url, params, user_agent)
def get_a_bogus_from_js(url: str, params: str, user_agent: str):
"""
通过js获取 a_bogus 参数
Get a_bogus parameter through js
Args:
url:
params:
@@ -84,8 +84,8 @@ def get_a_bogus_from_js(url: str, params: str, user_agent: str):
async def get_a_bogus_from_playright(params: str, post_data: dict, user_agent: str, page: Page):
"""
通过playright获取 a_bogus 参数
playwright版本已失效
Get a_bogus parameter through playwright
playwright version is deprecated
Returns:
"""
@@ -100,73 +100,73 @@ async def get_a_bogus_from_playright(params: str, post_data: dict, user_agent: s
def parse_video_info_from_url(url: str) -> VideoUrlInfo:
"""
从抖音视频URL中解析出视频ID
支持以下格式:
1. 普通视频链接: https://www.douyin.com/video/7525082444551310602
2. 带modal_id参数的链接:
Parse video ID from Douyin video URL
Supports the following formats:
1. Normal video link: https://www.douyin.com/video/7525082444551310602
2. Link with modal_id parameter:
- https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?modal_id=7525082444551310602
- https://www.douyin.com/root/search/python?modal_id=7471165520058862848
3. 短链接: https://v.douyin.com/iF12345ABC/ (需要client解析)
4. 纯ID: 7525082444551310602
3. Short link: https://v.douyin.com/iF12345ABC/ (requires client parsing)
4. Pure ID: 7525082444551310602
Args:
url: 抖音视频链接或ID
url: Douyin video link or ID
Returns:
VideoUrlInfo: 包含视频ID的对象
VideoUrlInfo: Object containing video ID
"""
# 如果是纯数字ID,直接返回
# If it's a pure numeric ID, return directly
if url.isdigit():
return VideoUrlInfo(aweme_id=url, url_type="normal")
# 检查是否是短链接 (v.douyin.com)
# Check if it's a short link (v.douyin.com)
if "v.douyin.com" in url or url.startswith("http") and len(url) < 50 and "video" not in url:
return VideoUrlInfo(aweme_id="", url_type="short") # 需要通过client解析
return VideoUrlInfo(aweme_id="", url_type="short") # Requires client parsing
# 尝试从URL参数中提取modal_id
# Try to extract modal_id from URL parameters
params = extract_url_params_to_dict(url)
modal_id = params.get("modal_id")
if modal_id:
return VideoUrlInfo(aweme_id=modal_id, url_type="modal")
# 从标准视频URL中提取ID: /video/数字
# Extract ID from standard video URL: /video/number
video_pattern = r'/video/(\d+)'
match = re.search(video_pattern, url)
if match:
aweme_id = match.group(1)
return VideoUrlInfo(aweme_id=aweme_id, url_type="normal")
raise ValueError(f"无法从URL中解析出视频ID: {url}")
raise ValueError(f"Unable to parse video ID from URL: {url}")
def parse_creator_info_from_url(url: str) -> CreatorUrlInfo:
"""
从抖音创作者主页URL中解析出创作者ID (sec_user_id)
支持以下格式:
1. 创作者主页: https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?from_tab_name=main
2. 纯ID: MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE
Parse creator ID (sec_user_id) from Douyin creator homepage URL
Supports the following formats:
1. Creator homepage: https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?from_tab_name=main
2. Pure ID: MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE
Args:
url: 抖音创作者主页链接或sec_user_id
url: Douyin creator homepage link or sec_user_id
Returns:
CreatorUrlInfo: 包含创作者ID的对象
CreatorUrlInfo: Object containing creator ID
"""
# 如果是纯ID格式(通常以MS4wLjABAAAA开头),直接返回
# If it's a pure ID format (usually starts with MS4wLjABAAAA), return directly
if url.startswith("MS4wLjABAAAA") or (not url.startswith("http") and "douyin.com" not in url):
return CreatorUrlInfo(sec_user_id=url)
# 从创作者主页URL中提取sec_user_id: /user/xxx
# Extract sec_user_id from creator homepage URL: /user/xxx
user_pattern = r'/user/([^/?]+)'
match = re.search(user_pattern, url)
if match:
sec_user_id = match.group(1)
return CreatorUrlInfo(sec_user_id=sec_user_id)
raise ValueError(f"无法从URL中解析出创作者ID: {url}")
raise ValueError(f"Unable to parse creator ID from URL: {url}")
if __name__ == '__main__':
# 测试视频URL解析
print("=== 视频URL解析测试 ===")
# Test video URL parsing
print("=== Video URL Parsing Test ===")
test_urls = [
"https://www.douyin.com/video/7525082444551310602",
"https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?from_tab_name=main&modal_id=7525082444551310602",
@@ -177,13 +177,13 @@ if __name__ == '__main__':
try:
result = parse_video_info_from_url(url)
print(f"✓ URL: {url[:80]}...")
print(f" 结果: {result}\n")
print(f" Result: {result}\n")
except Exception as e:
print(f"✗ URL: {url}")
print(f" 错误: {e}\n")
print(f" Error: {e}\n")
# 测试创作者URL解析
print("=== 创作者URL解析测试 ===")
# Test creator URL parsing
print("=== Creator URL Parsing Test ===")
test_creator_urls = [
"https://www.douyin.com/user/MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE?from_tab_name=main",
"MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE",
@@ -192,7 +192,7 @@ if __name__ == '__main__':
try:
result = parse_creator_info_from_url(url)
print(f"✓ URL: {url[:80]}...")
print(f" 结果: {result}\n")
print(f" Result: {result}\n")
except Exception as e:
print(f"✗ URL: {url}")
print(f" 错误: {e}\n")
print(f" Error: {e}\n")
media_platform/douyin/login.py
+23 -23
View File
@@ -53,7 +53,7 @@ class DouYinLogin(AbstractLogin):
async def begin(self):
"""
Start login douyin website
滑块中间页面的验证准确率不太OK... 如果没有特俗要求，建议不开抖音登录，或者使用cookies登录
The verification accuracy of the slider verification is not very good... If there are no special requirements, it is recommended not to use Douyin login, or use cookie login
"""
# popup login dialog
@@ -69,7 +69,7 @@ class DouYinLogin(AbstractLogin):
else:
raise ValueError("[DouYinLogin.begin] Invalid Login Type Currently only supported qrcode or phone or cookie ...")
# 如果页面重定向到滑动验证码页面，需要再次滑动滑块
# If the page redirects to the slider verification page, need to slide again
await asyncio.sleep(6)
current_page_title = await self.context_page.title()
if "验证码中间页" in current_page_title:
@@ -147,10 +147,10 @@ class DouYinLogin(AbstractLogin):
send_sms_code_btn = self.context_page.locator("xpath=//span[text() = '获取验证码']")
await send_sms_code_btn.click()
# 检查是否有滑动验证码
# Check if there is slider verification
await self.check_page_display_slider(move_step=10, slider_level="easy")
cache_client = CacheFactory.create_cache(config.CACHE_TYPE_MEMORY)
max_get_sms_code_time = 60 * 2 # 最长获取验证码的时间为2分钟
max_get_sms_code_time = 60 * 2 # Maximum time to get verification code is 2 minutes
while max_get_sms_code_time > 0:
utils.logger.info(f"[DouYinLogin.login_by_mobile] get douyin sms code from redis remaining time {max_get_sms_code_time}s ...")
await asyncio.sleep(1)
@@ -164,20 +164,20 @@ class DouYinLogin(AbstractLogin):
await sms_code_input_ele.fill(value=sms_code_value.decode())
await asyncio.sleep(0.5)
submit_btn_ele = self.context_page.locator("xpath=//button[@class='web-login-button']")
await submit_btn_ele.click() # 点击登录
# todo ... 应该还需要检查验证码的正确性有可能输入的验证码不正确
await submit_btn_ele.click() # Click login
# todo ... should also check the correctness of the verification code, it may be incorrect
break
async def check_page_display_slider(self, move_step: int = 10, slider_level: str = "easy"):
"""
检查页面是否出现滑动验证码
Check if slider verification appears on the page
:return:
"""
# 等待滑动验证码的出现
# Wait for slider verification to appear
back_selector = "#captcha-verify-image"
try:
await self.context_page.wait_for_selector(selector=back_selector, state="visible", timeout=30 * 1000)
except PlaywrightTimeoutError: # 没有滑动验证码，直接返回
except PlaywrightTimeoutError: # No slider verification, return directly
return
gap_selector = 'xpath=//*[@id="captcha_container"]/div/div[2]/img[2]'
@@ -191,16 +191,16 @@ class DouYinLogin(AbstractLogin):
await self.move_slider(back_selector, gap_selector, move_step, slider_level)
await asyncio.sleep(1)
# 如果滑块滑动慢了，或者验证失败了，会提示操作过慢，这里点一下刷新按钮
# If the slider is too slow or verification failed, it will prompt "操作过慢", click the refresh button here
page_content = await self.context_page.content()
if "操作过慢" in page_content or "提示重新操作" in page_content:
utils.logger.info("[DouYinLogin.check_page_display_slider] slider verify failed, retry ...")
await self.context_page.click(selector="//a[contains(@class, 'secsdk_captcha_refresh')]")
continue
# 滑动成功后，等待滑块消失
# After successful sliding, wait for the slider to disappear
await self.context_page.wait_for_selector(selector=back_selector, state="hidden", timeout=1000)
# 如果滑块消失了，说明验证成功了，跳出循环，如果没有消失，说明验证失败了，上面这一行代码会抛出异常被捕获后继续循环滑动验证码
# If the slider disappears, it means the verification is successful, break the loop. If not, it means the verification failed, the above line will throw an exception and be caught to continue the loop
utils.logger.info("[DouYinLogin.check_page_display_slider] slider verify success ...")
slider_verify_success = True
except Exception as e:
@@ -213,10 +213,10 @@ class DouYinLogin(AbstractLogin):
async def move_slider(self, back_selector: str, gap_selector: str, move_step: int = 10, slider_level="easy"):
"""
Move the slider to the right to complete the verification
:param back_selector: 滑动验证码背景图片的选择器
:param gap_selector: 滑动验证码的滑块选择器
:param move_step: 是控制单次移动速度的比例是1/10 默认是1 相当于 传入的这个距离不管多远0.1秒钟移动完 越大越慢
:param slider_level: 滑块难度 easy hard,分别对应手机验证码的滑块和验证码中间的滑块
:param back_selector: Selector for the slider verification background image
:param gap_selector: Selector for the slider verification slider
:param move_step: Controls the ratio of single movement speed, default is 1, meaning the distance moves in 0.1 seconds no matter how far, larger value means slower
:param slider_level: Slider difficulty easy hard, corresponding to the slider for mobile verification code and the slider in the middle of verification code
:return:
"""
@@ -234,31 +234,31 @@ class DouYinLogin(AbstractLogin):
)
gap_src = str(await gap_elements.get_property("src")) # type: ignore
# 识别滑块位置
# Identify slider position
slide_app = utils.Slide(gap=gap_src, bg=slide_back)
distance = slide_app.discern()
# 获取移动轨迹
# Get movement trajectory
tracks = utils.get_tracks(distance, slider_level)
new_1 = tracks[-1] - (sum(tracks) - distance)
tracks.pop()
tracks.append(new_1)
# 根据轨迹拖拽滑块到指定位置
# Drag slider to specified position according to trajectory
element = await self.context_page.query_selector(gap_selector)
bounding_box = await element.bounding_box() # type: ignore
await self.context_page.mouse.move(bounding_box["x"] + bounding_box["width"] / 2, # type: ignore
bounding_box["y"] + bounding_box["height"] / 2) # type: ignore
# 这里获取到x坐标中心点位置
# Get x coordinate center position
x = bounding_box["x"] + bounding_box["width"] / 2 # type: ignore
# 模拟滑动操作
# Simulate sliding operation
await element.hover() # type: ignore
await self.context_page.mouse.down()
for track in tracks:
# 循环鼠标按照轨迹移动
# steps 是控制单次移动速度的比例是1/10 默认是1 相当于 传入的这个距离不管多远0.1秒钟移动完 越大越慢
# Loop mouse movement according to trajectory
# steps controls the ratio of single movement speed, default is 1, meaning the distance moves in 0.1 seconds no matter how far, larger value means slower
await self.context_page.mouse.move(x + track, 0, steps=move_step)
x += track
await self.context_page.mouse.up()
media_platform/kuaishou/client.py
+13 -13
View File
@@ -57,11 +57,11 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
self.playwright_page = playwright_page
self.cookie_dict = cookie_dict
self.graphql = KuaiShouGraphQL()
# 初始化代理池（来自 ProxyRefreshMixin）
# Initialize proxy pool (from ProxyRefreshMixin)
self.init_proxy_pool(proxy_ip_pool)
async def request(self, method, url, **kwargs) -> Any:
# 每次请求前检测代理是否过期
# Check if proxy is expired before each request
await self._refresh_proxy_if_expired()
async with httpx.AsyncClient(proxy=self.proxy) as client:
@@ -222,7 +222,7 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
comments = vision_commen_list.get("rootComments", [])
if len(result) + len(comments) > max_count:
comments = comments[: max_count - len(result)]
if callback: # 如果有回调函数，就执行回调函数
if callback: # If there is a callback function, execute the callback function
await callback(photo_id, comments)
result.extend(comments)
await asyncio.sleep(crawl_interval)
@@ -240,12 +240,12 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
callback: Optional[Callable] = None,
) -> List[Dict]:
"""
获取指定一级评论下的所有二级评论, 该方法会一直查找一级评论下的所有二级评论信息
Get all second-level comments under specified first-level comments, this method will continue to find all second-level comment information under first-level comments
Args:
comments: 评论列表
photo_id: 视频id
crawl_interval: 爬取一次评论的延迟单位（秒）
callback: 一次评论爬取结束后
comments: Comment list
photo_id: Video ID
crawl_interval: Delay unit for crawling comments once (seconds)
callback: Callback after one comment crawl ends
Returns:
"""
@@ -285,7 +285,7 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
async def get_creator_info(self, user_id: str) -> Dict:
"""
eg: https://www.kuaishou.com/profile/3x4jtnbfter525a
快手用户主页
Kuaishou user homepage
"""
visionProfile = await self.get_creator_profile(user_id)
@@ -298,11 +298,11 @@ class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
callback: Optional[Callable] = None,
) -> List[Dict]:
"""
获取指定用户下的所有发过的帖子，该方法会一直查找一个用户下的所有帖子信息
Get all posts published by the specified user, this method will continue to find all post information under a user
Args:
user_id: 用户ID
crawl_interval: 爬取一次的延迟单位（秒）
callback: 一次分页爬取结束后的更新回调函数
user_id: User ID
crawl_interval: Delay unit for crawling once (seconds)
callback: Update callback function after one page crawl ends
Returns:
"""
media_platform/kuaishou/core.py
+12 -12
View File
@@ -58,7 +58,7 @@ class KuaishouCrawler(AbstractCrawler):
self.index_url = "https://www.kuaishou.com"
self.user_agent = utils.get_user_agent()
self.cdp_manager = None
self.ip_proxy_pool = None # 代理IP池，用于代理自动刷新
self.ip_proxy_pool = None # Proxy IP pool, used for automatic proxy refresh
async def start(self):
playwright_proxy_format, httpx_proxy_format = None, None
@@ -72,9 +72,9 @@ class KuaishouCrawler(AbstractCrawler):
)
async with async_playwright() as playwright:
# 根据配置选择启动模式
# Select startup mode based on configuration
if config.ENABLE_CDP_MODE:
utils.logger.info("[KuaishouCrawler] 使用CDP模式启动浏览器")
utils.logger.info("[KuaishouCrawler] Launching browser using CDP mode")
self.browser_context = await self.launch_browser_with_cdp(
playwright,
playwright_proxy_format,
@@ -82,7 +82,7 @@ class KuaishouCrawler(AbstractCrawler):
headless=config.CDP_HEADLESS,
)
else:
utils.logger.info("[KuaishouCrawler] 使用标准模式启动浏览器")
utils.logger.info("[KuaishouCrawler] Launching browser using standard mode")
# Launch a browser context.
chromium = playwright.chromium
self.browser_context = await self.launch_browser(
@@ -318,7 +318,7 @@ class KuaishouCrawler(AbstractCrawler):
},
playwright_page=self.context_page,
cookie_dict=cookie_dict,
proxy_ip_pool=self.ip_proxy_pool, # 传递代理池用于自动刷新
proxy_ip_pool=self.ip_proxy_pool, # Pass proxy pool for automatic refresh
)
return ks_client_obj
@@ -344,7 +344,7 @@ class KuaishouCrawler(AbstractCrawler):
proxy=playwright_proxy, # type: ignore
viewport={"width": 1920, "height": 1080},
user_agent=user_agent,
channel="chrome", # 使用系统的Chrome稳定版
channel="chrome", # Use system's stable Chrome version
)
return browser_context
else:
@@ -362,7 +362,7 @@ class KuaishouCrawler(AbstractCrawler):
headless: bool = True,
) -> BrowserContext:
"""
使用CDP模式启动浏览器
Launch browser using CDP mode
"""
try:
self.cdp_manager = CDPBrowserManager()
@@ -373,17 +373,17 @@ class KuaishouCrawler(AbstractCrawler):
headless=headless,
)
# 显示浏览器信息
# Display browser information
browser_info = await self.cdp_manager.get_browser_info()
utils.logger.info(f"[KuaishouCrawler] CDP浏览器信息: {browser_info}")
utils.logger.info(f"[KuaishouCrawler] CDP browser info: {browser_info}")
return browser_context
except Exception as e:
utils.logger.error(
f"[KuaishouCrawler] CDP模式启动失败，回退到标准模式: {e}"
f"[KuaishouCrawler] CDP mode launch failed, fallback to standard mode: {e}"
)
# 回退到标准模式
# Fallback to standard mode
chromium = playwright.chromium
return await self.launch_browser(
chromium, playwright_proxy, user_agent, headless
@@ -438,7 +438,7 @@ class KuaishouCrawler(AbstractCrawler):
async def close(self):
"""Close browser context"""
# 如果使用CDP模式，需要特殊处理
# If using CDP mode, need special handling
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
media_platform/kuaishou/graphql.py
+2 -2
View File
@@ -18,8 +18,8 @@
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
# 快手的数据传输是基于GraphQL实现的
# 这个类负责获取一些GraphQL的schema
# Kuaishou's data transmission is based on GraphQL
# This class is responsible for obtaining some GraphQL schemas
from typing import Dict
media_platform/kuaishou/help.py
+26 -26
View File
@@ -26,59 +26,59 @@ from model.m_kuaishou import VideoUrlInfo, CreatorUrlInfo
def parse_video_info_from_url(url: str) -> VideoUrlInfo:
"""
从快手视频URL中解析出视频ID
支持以下格式:
1. 完整视频URL: "https://www.kuaishou.com/short-video/3x3zxz4mjrsc8ke?authorId=3x84qugg4ch9zhs&streamSource=search"
2. 纯视频ID: "3x3zxz4mjrsc8ke"
Parse video ID from Kuaishou video URL
Supports the following formats:
1. Full video URL: "https://www.kuaishou.com/short-video/3x3zxz4mjrsc8ke?authorId=3x84qugg4ch9zhs&streamSource=search"
2. Pure video ID: "3x3zxz4mjrsc8ke"
Args:
url: 快手视频链接或视频ID
url: Kuaishou video link or video ID
Returns:
VideoUrlInfo: 包含视频ID的对象
VideoUrlInfo: Object containing video ID
"""
# 如果不包含http且不包含kuaishou.com，认为是纯ID
# If it doesn't contain http and doesn't contain kuaishou.com, consider it as pure ID
if not url.startswith("http") and "kuaishou.com" not in url:
return VideoUrlInfo(video_id=url, url_type="normal")
# 从标准视频URL中提取ID: /short-video/视频ID
# Extract ID from standard video URL: /short-video/video_ID
video_pattern = r'/short-video/([a-zA-Z0-9_-]+)'
match = re.search(video_pattern, url)
if match:
video_id = match.group(1)
return VideoUrlInfo(video_id=video_id, url_type="normal")
raise ValueError(f"无法从URL中解析出视频ID: {url}")
raise ValueError(f"Unable to parse video ID from URL: {url}")
def parse_creator_info_from_url(url: str) -> CreatorUrlInfo:
"""
从快手创作者主页URL中解析出创作者ID
支持以下格式:
1. 创作者主页: "https://www.kuaishou.com/profile/3x84qugg4ch9zhs"
2. 纯ID: "3x4sm73aye7jq7i"
Parse creator ID from Kuaishou creator homepage URL
Supports the following formats:
1. Creator homepage: "https://www.kuaishou.com/profile/3x84qugg4ch9zhs"
2. Pure ID: "3x4sm73aye7jq7i"
Args:
url: 快手创作者主页链接或user_id
url: Kuaishou creator homepage link or user_id
Returns:
CreatorUrlInfo: 包含创作者ID的对象
CreatorUrlInfo: Object containing creator ID
"""
# 如果不包含http且不包含kuaishou.com，认为是纯ID
# If it doesn't contain http and doesn't contain kuaishou.com, consider it as pure ID
if not url.startswith("http") and "kuaishou.com" not in url:
return CreatorUrlInfo(user_id=url)
# 从创作者主页URL中提取user_id: /profile/xxx
# Extract user_id from creator homepage URL: /profile/xxx
user_pattern = r'/profile/([a-zA-Z0-9_-]+)'
match = re.search(user_pattern, url)
if match:
user_id = match.group(1)
return CreatorUrlInfo(user_id=user_id)
raise ValueError(f"无法从URL中解析出创作者ID: {url}")
raise ValueError(f"Unable to parse creator ID from URL: {url}")
if __name__ == '__main__':
# 测试视频URL解析
print("=== 视频URL解析测试 ===")
# Test video URL parsing
print("=== Video URL Parsing Test ===")
test_video_urls = [
"https://www.kuaishou.com/short-video/3x3zxz4mjrsc8ke?authorId=3x84qugg4ch9zhs&streamSource=search&area=searchxxnull&searchKey=python",
"3xf8enb8dbj6uig",
@@ -87,13 +87,13 @@ if __name__ == '__main__':
try:
result = parse_video_info_from_url(url)
print(f"✓ URL: {url[:80]}...")
print(f" 结果: {result}\n")
print(f" Result: {result}\n")
except Exception as e:
print(f"✗ URL: {url}")
print(f" 错误: {e}\n")
print(f" Error: {e}\n")
# 测试创作者URL解析
print("=== 创作者URL解析测试 ===")
# Test creator URL parsing
print("=== Creator URL Parsing Test ===")
test_creator_urls = [
"https://www.kuaishou.com/profile/3x84qugg4ch9zhs",
"3x4sm73aye7jq7i",
@@ -102,7 +102,7 @@ if __name__ == '__main__':
try:
result = parse_creator_info_from_url(url)
print(f"✓ URL: {url[:80]}...")
print(f" 结果: {result}\n")
print(f" Result: {result}\n")
except Exception as e:
print(f"✗ URL: {url}")
print(f" 错误: {e}\n")
print(f" Error: {e}\n")
media_platform/tieba/client.py
+141 -141
View File
@@ -48,7 +48,7 @@ class BaiduTieBaClient(AbstractApiClient):
):
self.ip_pool: Optional[ProxyIpPool] = ip_pool
self.timeout = timeout
# 使用传入的headers(包含真实浏览器UA)或默认headers
# Use provided headers (including real browser UA) or default headers
self.headers = headers or {
"User-Agent": utils.get_user_agent(),
"Cookie": "",
@@ -56,21 +56,21 @@ class BaiduTieBaClient(AbstractApiClient):
self._host = "https://tieba.baidu.com"
self._page_extractor = TieBaExtractor()
self.default_ip_proxy = default_ip_proxy
self.playwright_page = playwright_page # Playwright页面对象
self.playwright_page = playwright_page # Playwright page object
def _sync_request(self, method, url, proxy=None, **kwargs):
"""
同步的requests请求方法
Synchronous requests method
Args:
method: 请求方法
url: 请求的URL
proxy: 代理IP
**kwargs: 其他请求参数
method: Request method
url: Request URL
proxy: Proxy IP
**kwargs: Other request parameters
Returns:
response对象
Response object
"""
# 构造代理字典
# Construct proxy dictionary
proxies = None
if proxy:
proxies = {
@@ -78,7 +78,7 @@ class BaiduTieBaClient(AbstractApiClient):
"https": proxy,
}
# 发送请求
# Send request
response = requests.request(
method=method,
url=url,
@@ -91,7 +91,7 @@ class BaiduTieBaClient(AbstractApiClient):
async def _refresh_proxy_if_expired(self) -> None:
"""
检测代理是否过期，如果过期则自动刷新
Check if proxy is expired and automatically refresh if necessary
"""
if self.ip_pool is None:
return
@@ -101,7 +101,7 @@ class BaiduTieBaClient(AbstractApiClient):
"[BaiduTieBaClient._refresh_proxy_if_expired] Proxy expired, refreshing..."
)
new_proxy = await self.ip_pool.get_or_refresh_proxy()
# 更新代理URL
# Update proxy URL
_, self.default_ip_proxy = utils.format_proxy_info(new_proxy)
utils.logger.info(
f"[BaiduTieBaClient._refresh_proxy_if_expired] New proxy: {new_proxy.ip}:{new_proxy.port}"
@@ -110,23 +110,23 @@ class BaiduTieBaClient(AbstractApiClient):
@retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
async def request(self, method, url, return_ori_content=False, proxy=None, **kwargs) -> Union[str, Any]:
"""
封装requests的公共请求方法，对请求响应做一些处理
Common request method wrapper for requests, handles request responses
Args:
method: 请求方法
url: 请求的URL
return_ori_content: 是否返回原始内容
proxy: 代理IP
**kwargs: 其他请求参数，例如请求头、请求体等
method: Request method
url: Request URL
return_ori_content: Whether to return original content
proxy: Proxy IP
**kwargs: Other request parameters, such as headers, request body, etc.
Returns:
"""
# 每次请求前检测代理是否过期
# Check if proxy is expired before each request
await self._refresh_proxy_if_expired()
actual_proxy = proxy if proxy else self.default_ip_proxy
# 在线程池中执行同步的requests请求
# Execute synchronous requests in thread pool
response = await asyncio.to_thread(
self._sync_request,
method,
@@ -151,11 +151,11 @@ class BaiduTieBaClient(AbstractApiClient):
async def get(self, uri: str, params=None, return_ori_content=False, **kwargs) -> Any:
"""
GET请求，对请求头签名
GET request with header signing
Args:
uri: 请求路由
params: 请求参数
return_ori_content: 是否返回原始内容
uri: Request route
params: Request parameters
return_ori_content: Whether to return original content
Returns:
@@ -175,15 +175,15 @@ class BaiduTieBaClient(AbstractApiClient):
self.default_ip_proxy = proxy
return res
utils.logger.error(f"[BaiduTieBaClient.get] 达到了最大重试次数，IP已经被Block，请尝试更换新的IP代理: {e}")
raise Exception(f"[BaiduTieBaClient.get] 达到了最大重试次数，IP已经被Block，请尝试更换新的IP代理: {e}")
utils.logger.error(f"[BaiduTieBaClient.get] Reached maximum retry attempts, IP is blocked, please try a new IP proxy: {e}")
raise Exception(f"[BaiduTieBaClient.get] Reached maximum retry attempts, IP is blocked, please try a new IP proxy: {e}")
async def post(self, uri: str, data: dict, **kwargs) -> Dict:
"""
POST请求，对请求头签名
POST request with header signing
Args:
uri: 请求路由
data: 请求体参数
uri: Request route
data: Request body parameters
Returns:
@@ -193,13 +193,13 @@ class BaiduTieBaClient(AbstractApiClient):
async def pong(self, browser_context: BrowserContext = None) -> bool:
"""
用于检查登录态是否失效了
使用Cookie检测而非API调用,避免被检测
Check if login state is still valid
Uses Cookie detection instead of API calls to avoid detection
Args:
browser_context: 浏览器上下文对象
browser_context: Browser context object
Returns:
bool: True表示已登录,False表示未登录
bool: True if logged in, False if not logged in
"""
utils.logger.info("[BaiduTieBaClient.pong] Begin to check tieba login state by cookies...")
@@ -208,13 +208,13 @@ class BaiduTieBaClient(AbstractApiClient):
return False
try:
# 从浏览器获取cookies并检查关键登录cookie
# Get cookies from browser and check key login cookies
_, cookie_dict = utils.convert_cookies(await browser_context.cookies())
# 百度贴吧的登录标识: STOKEN 或 PTOKEN
# Baidu Tieba login identifiers: STOKEN or PTOKEN
stoken = cookie_dict.get("STOKEN")
ptoken = cookie_dict.get("PTOKEN")
bduss = cookie_dict.get("BDUSS") # 百度通用登录cookie
bduss = cookie_dict.get("BDUSS") # Baidu universal login cookie
if stoken or ptoken or bduss:
utils.logger.info(f"[BaiduTieBaClient.pong] Login state verified by cookies (STOKEN: {bool(stoken)}, PTOKEN: {bool(ptoken)}, BDUSS: {bool(bduss)})")
@@ -229,9 +229,9 @@ class BaiduTieBaClient(AbstractApiClient):
async def update_cookies(self, browser_context: BrowserContext):
"""
API客户端提供的更新cookies方法，一般情况下登录成功后会调用此方法
Update cookies method provided by API client, usually called after successful login
Args:
browser_context: 浏览器上下文对象
browser_context: Browser context object
Returns:
@@ -249,13 +249,13 @@ class BaiduTieBaClient(AbstractApiClient):
note_type: SearchNoteType = SearchNoteType.FIXED_THREAD,
) -> List[TiebaNote]:
"""
根据关键词搜索贴吧帖子 (使用Playwright访问页面,避免API检测)
Search Tieba posts by keyword (uses Playwright to access page, avoiding API detection)
Args:
keyword: 关键词
page: 分页第几页
page_size: 每页大小
sort: 结果排序方式
note_type: 帖子类型（主题贴｜主题+回复混合模式）
keyword: Keyword
page: Page number
page_size: Page size
sort: Result sort method
note_type: Post type (main thread | main thread + reply mixed mode)
Returns:
"""
@@ -263,8 +263,8 @@ class BaiduTieBaClient(AbstractApiClient):
utils.logger.error("[BaiduTieBaClient.get_notes_by_keyword] playwright_page is None, cannot use browser mode")
raise Exception("playwright_page is required for browser-based search")
# 构造搜索URL
# 示例: https://tieba.baidu.com/f/search/res?ie=utf-8&qw=编程
# Construct search URL
# Example: https://tieba.baidu.com/f/search/res?ie=utf-8&qw=keyword
search_url = f"{self._host}/f/search/res"
params = {
"ie": "utf-8",
@@ -275,64 +275,64 @@ class BaiduTieBaClient(AbstractApiClient):
"only_thread": note_type.value,
}
# 拼接完整URL
# Concatenate full URL
full_url = f"{search_url}?{urlencode(params)}"
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] 访问搜索页面: {full_url}")
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] Accessing search page: {full_url}")
try:
# 使用Playwright访问搜索页面
# Use Playwright to access search page
await self.playwright_page.goto(full_url, wait_until="domcontentloaded")
# 等待页面加载,使用配置文件中的延时设置
# Wait for page loading, using delay setting from config file
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
# 获取页面HTML内容
# Get page HTML content
page_content = await self.playwright_page.content()
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] 成功获取搜索页面HTML,长度: {len(page_content)}")
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] Successfully retrieved search page HTML, length: {len(page_content)}")
# 提取搜索结果
# Extract search results
notes = self._page_extractor.extract_search_note_list(page_content)
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] 提取到 {len(notes)} 条帖子")
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_keyword] Extracted {len(notes)} posts")
return notes
except Exception as e:
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_keyword] 搜索失败: {e}")
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_keyword] Search failed: {e}")
raise
async def get_note_by_id(self, note_id: str) -> TiebaNote:
"""
根据帖子ID获取帖子详情 (使用Playwright访问页面,避免API检测)
Get post details by post ID (uses Playwright to access page, avoiding API detection)
Args:
note_id: 帖子ID
note_id: Post ID
Returns:
TiebaNote: 帖子详情对象
TiebaNote: Post detail object
"""
if not self.playwright_page:
utils.logger.error("[BaiduTieBaClient.get_note_by_id] playwright_page is None, cannot use browser mode")
raise Exception("playwright_page is required for browser-based note detail fetching")
# 构造帖子详情URL
# Construct post detail URL
note_url = f"{self._host}/p/{note_id}"
utils.logger.info(f"[BaiduTieBaClient.get_note_by_id] 访问帖子详情页面: {note_url}")
utils.logger.info(f"[BaiduTieBaClient.get_note_by_id] Accessing post detail page: {note_url}")
try:
# 使用Playwright访问帖子详情页面
# Use Playwright to access post detail page
await self.playwright_page.goto(note_url, wait_until="domcontentloaded")
# 等待页面加载,使用配置文件中的延时设置
# Wait for page loading, using delay setting from config file
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
# 获取页面HTML内容
# Get page HTML content
page_content = await self.playwright_page.content()
utils.logger.info(f"[BaiduTieBaClient.get_note_by_id] 成功获取帖子详情HTML,长度: {len(page_content)}")
utils.logger.info(f"[BaiduTieBaClient.get_note_by_id] Successfully retrieved post detail HTML, length: {len(page_content)}")
# 提取帖子详情
# Extract post details
note_detail = self._page_extractor.extract_note_detail(page_content)
return note_detail
except Exception as e:
utils.logger.error(f"[BaiduTieBaClient.get_note_by_id] 获取帖子详情失败: {e}")
utils.logger.error(f"[BaiduTieBaClient.get_note_by_id] Failed to get post details: {e}")
raise
async def get_note_all_comments(
@@ -343,14 +343,14 @@ class BaiduTieBaClient(AbstractApiClient):
max_count: int = 10,
) -> List[TiebaComment]:
"""
获取指定帖子下的所有一级评论 (使用Playwright访问页面,避免API检测)
Get all first-level comments for specified post (uses Playwright to access page, avoiding API detection)
Args:
note_detail: 帖子详情对象
crawl_interval: 爬取一次笔记的延迟单位（秒）
callback: 一次笔记爬取结束后的回调函数
max_count: 一次帖子爬取的最大评论数量
note_detail: Post detail object
crawl_interval: Crawl delay interval in seconds
callback: Callback function after one post crawl completes
max_count: Maximum number of comments to crawl per post
Returns:
List[TiebaComment]: 评论列表
List[TiebaComment]: Comment list
"""
if not self.playwright_page:
utils.logger.error("[BaiduTieBaClient.get_note_all_comments] playwright_page is None, cannot use browser mode")
@@ -360,30 +360,30 @@ class BaiduTieBaClient(AbstractApiClient):
current_page = 1
while note_detail.total_replay_page >= current_page and len(result) < max_count:
# 构造评论页URL
# Construct comment page URL
comment_url = f"{self._host}/p/{note_detail.note_id}?pn={current_page}"
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] 访问评论页面: {comment_url}")
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] Accessing comment page: {comment_url}")
try:
# 使用Playwright访问评论页面
# Use Playwright to access comment page
await self.playwright_page.goto(comment_url, wait_until="domcontentloaded")
# 等待页面加载,使用配置文件中的延时设置
# Wait for page loading, using delay setting from config file
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
# 获取页面HTML内容
# Get page HTML content
page_content = await self.playwright_page.content()
# 提取评论
# Extract comments
comments = self._page_extractor.extract_tieba_note_parment_comments(
page_content, note_id=note_detail.note_id
)
if not comments:
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] 第{current_page}页没有评论,停止爬取")
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] Page {current_page} has no comments, stopping crawl")
break
# 限制评论数量
# Limit comment count
if len(result) + len(comments) > max_count:
comments = comments[:max_count - len(result)]
@@ -392,7 +392,7 @@ class BaiduTieBaClient(AbstractApiClient):
result.extend(comments)
# 获取所有子评论
# Get all sub-comments
await self.get_comments_all_sub_comments(
comments, crawl_interval=crawl_interval, callback=callback
)
@@ -401,10 +401,10 @@ class BaiduTieBaClient(AbstractApiClient):
current_page += 1
except Exception as e:
utils.logger.error(f"[BaiduTieBaClient.get_note_all_comments] 获取第{current_page}页评论失败: {e}")
utils.logger.error(f"[BaiduTieBaClient.get_note_all_comments] Failed to get page {current_page} comments: {e}")
break
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] 共获取 {len(result)} 条一级评论")
utils.logger.info(f"[BaiduTieBaClient.get_note_all_comments] Total retrieved {len(result)} first-level comments")
return result
async def get_comments_all_sub_comments(
@@ -414,14 +414,14 @@ class BaiduTieBaClient(AbstractApiClient):
callback: Optional[Callable] = None,
) -> List[TiebaComment]:
"""
获取指定评论下的所有子评论 (使用Playwright访问页面,避免API检测)
Get all sub-comments for specified comments (uses Playwright to access page, avoiding API detection)
Args:
comments: 评论列表
crawl_interval: 爬取一次笔记的延迟单位（秒）
callback: 一次笔记爬取结束后的回调函数
comments: Comment list
crawl_interval: Crawl delay interval in seconds
callback: Callback function after one post crawl completes
Returns:
List[TiebaComment]: 子评论列表
List[TiebaComment]: Sub-comment list
"""
if not config.ENABLE_GET_SUB_COMMENTS:
return []
@@ -440,7 +440,7 @@ class BaiduTieBaClient(AbstractApiClient):
max_sub_page_num = parment_comment.sub_comment_count // 10 + 1
while max_sub_page_num >= current_page:
# 构造子评论URL
# Construct sub-comment URL
sub_comment_url = (
f"{self._host}/p/comment?"
f"tid={parment_comment.note_id}&"
@@ -448,19 +448,19 @@ class BaiduTieBaClient(AbstractApiClient):
f"fid={parment_comment.tieba_id}&"
f"pn={current_page}"
)
utils.logger.info(f"[BaiduTieBaClient.get_comments_all_sub_comments] 访问子评论页面: {sub_comment_url}")
utils.logger.info(f"[BaiduTieBaClient.get_comments_all_sub_comments] Accessing sub-comment page: {sub_comment_url}")
try:
# 使用Playwright访问子评论页面
# Use Playwright to access sub-comment page
await self.playwright_page.goto(sub_comment_url, wait_until="domcontentloaded")
# 等待页面加载,使用配置文件中的延时设置
# Wait for page loading, using delay setting from config file
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
# 获取页面HTML内容
# Get page HTML content
page_content = await self.playwright_page.content()
# 提取子评论
# Extract sub-comments
sub_comments = self._page_extractor.extract_tieba_note_sub_comments(
page_content, parent_comment=parment_comment
)
@@ -468,7 +468,7 @@ class BaiduTieBaClient(AbstractApiClient):
if not sub_comments:
utils.logger.info(
f"[BaiduTieBaClient.get_comments_all_sub_comments] "
f"评论{parment_comment.comment_id}第{current_page}页没有子评论,停止爬取"
f"Comment {parment_comment.comment_id} page {current_page} has no sub-comments, stopping crawl"
)
break
@@ -482,125 +482,125 @@ class BaiduTieBaClient(AbstractApiClient):
except Exception as e:
utils.logger.error(
f"[BaiduTieBaClient.get_comments_all_sub_comments] "
f"获取评论{parment_comment.comment_id}第{current_page}页子评论失败: {e}"
f"Failed to get comment {parment_comment.comment_id} page {current_page} sub-comments: {e}"
)
break
utils.logger.info(f"[BaiduTieBaClient.get_comments_all_sub_comments] 共获取 {len(all_sub_comments)} 条子评论")
utils.logger.info(f"[BaiduTieBaClient.get_comments_all_sub_comments] Total retrieved {len(all_sub_comments)} sub-comments")
return all_sub_comments
async def get_notes_by_tieba_name(self, tieba_name: str, page_num: int) -> List[TiebaNote]:
"""
根据贴吧名称获取帖子列表 (使用Playwright访问页面,避免API检测)
Get post list by Tieba name (uses Playwright to access page, avoiding API detection)
Args:
tieba_name: 贴吧名称
page_num: 分页页码
tieba_name: Tieba name
page_num: Page number
Returns:
List[TiebaNote]: 帖子列表
List[TiebaNote]: Post list
"""
if not self.playwright_page:
utils.logger.error("[BaiduTieBaClient.get_notes_by_tieba_name] playwright_page is None, cannot use browser mode")
raise Exception("playwright_page is required for browser-based tieba note fetching")
# 构造贴吧帖子列表URL
# Construct Tieba post list URL
tieba_url = f"{self._host}/f?kw={quote(tieba_name)}&pn={page_num}"
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] 访问贴吧页面: {tieba_url}")
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] Accessing Tieba page: {tieba_url}")
try:
# 使用Playwright访问贴吧页面
# Use Playwright to access Tieba page
await self.playwright_page.goto(tieba_url, wait_until="domcontentloaded")
# 等待页面加载,使用配置文件中的延时设置
# Wait for page loading, using delay setting from config file
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
# 获取页面HTML内容
# Get page HTML content
page_content = await self.playwright_page.content()
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] 成功获取贴吧页面HTML,长度: {len(page_content)}")
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] Successfully retrieved Tieba page HTML, length: {len(page_content)}")
# 提取帖子列表
# Extract post list
notes = self._page_extractor.extract_tieba_note_list(page_content)
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] 提取到 {len(notes)} 条帖子")
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_tieba_name] Extracted {len(notes)} posts")
return notes
except Exception as e:
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_tieba_name] 获取贴吧帖子列表失败: {e}")
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_tieba_name] Failed to get Tieba post list: {e}")
raise
async def get_creator_info_by_url(self, creator_url: str) -> str:
"""
根据创作者URL获取创作者信息 (使用Playwright访问页面,避免API检测)
Get creator information by creator URL (uses Playwright to access page, avoiding API detection)
Args:
creator_url: 创作者主页URL
creator_url: Creator homepage URL
Returns:
str: 页面HTML内容
str: Page HTML content
"""
if not self.playwright_page:
utils.logger.error("[BaiduTieBaClient.get_creator_info_by_url] playwright_page is None, cannot use browser mode")
raise Exception("playwright_page is required for browser-based creator info fetching")
utils.logger.info(f"[BaiduTieBaClient.get_creator_info_by_url] 访问创作者主页: {creator_url}")
utils.logger.info(f"[BaiduTieBaClient.get_creator_info_by_url] Accessing creator homepage: {creator_url}")
try:
# 使用Playwright访问创作者主页
# Use Playwright to access creator homepage
await self.playwright_page.goto(creator_url, wait_until="domcontentloaded")
# 等待页面加载,使用配置文件中的延时设置
# Wait for page loading, using delay setting from config file
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
# 获取页面HTML内容
# Get page HTML content
page_content = await self.playwright_page.content()
utils.logger.info(f"[BaiduTieBaClient.get_creator_info_by_url] 成功获取创作者主页HTML,长度: {len(page_content)}")
utils.logger.info(f"[BaiduTieBaClient.get_creator_info_by_url] Successfully retrieved creator homepage HTML, length: {len(page_content)}")
return page_content
except Exception as e:
utils.logger.error(f"[BaiduTieBaClient.get_creator_info_by_url] 获取创作者主页失败: {e}")
utils.logger.error(f"[BaiduTieBaClient.get_creator_info_by_url] Failed to get creator homepage: {e}")
raise
async def get_notes_by_creator(self, user_name: str, page_number: int) -> Dict:
"""
根据创作者获取创作者的帖子 (使用Playwright访问页面,避免API检测)
Get creator's posts by creator (uses Playwright to access page, avoiding API detection)
Args:
user_name: 创作者用户名
page_number: 页码
user_name: Creator username
page_number: Page number
Returns:
Dict: 包含帖子数据的字典
Dict: Dictionary containing post data
"""
if not self.playwright_page:
utils.logger.error("[BaiduTieBaClient.get_notes_by_creator] playwright_page is None, cannot use browser mode")
raise Exception("playwright_page is required for browser-based creator notes fetching")
# 构造创作者帖子列表URL
# Construct creator post list URL
creator_url = f"{self._host}/home/get/getthread?un={quote(user_name)}&pn={page_number}&id=utf-8&_={utils.get_current_timestamp()}"
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_creator] 访问创作者帖子列表: {creator_url}")
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_creator] Accessing creator post list: {creator_url}")
try:
# 使用Playwright访问创作者帖子列表页面
# Use Playwright to access creator post list page
await self.playwright_page.goto(creator_url, wait_until="domcontentloaded")
# 等待页面加载,使用配置文件中的延时设置
# Wait for page loading, using delay setting from config file
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
# 获取页面内容(这个接口返回JSON)
# Get page content (this API returns JSON)
page_content = await self.playwright_page.content()
# 提取JSON数据(页面会包含<pre>标签或直接是JSON)
# Extract JSON data (page will contain <pre> tag or is directly JSON)
try:
# 尝试从页面中提取JSON
# Try to extract JSON from page
json_text = await self.playwright_page.evaluate("() => document.body.innerText")
result = json.loads(json_text)
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_creator] 成功获取创作者帖子数据")
utils.logger.info(f"[BaiduTieBaClient.get_notes_by_creator] Successfully retrieved creator post data")
return result
except json.JSONDecodeError as e:
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] JSON解析失败: {e}")
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] 页面内容: {page_content[:500]}")
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] JSON parsing failed: {e}")
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] Page content: {page_content[:500]}")
raise Exception(f"Failed to parse JSON from creator notes page: {e}")
except Exception as e:
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] 获取创作者帖子列表失败: {e}")
utils.logger.error(f"[BaiduTieBaClient.get_notes_by_creator] Failed to get creator post list: {e}")
raise
async def get_all_notes_by_creator_user_name(
@@ -612,18 +612,18 @@ class BaiduTieBaClient(AbstractApiClient):
creator_page_html_content: str = None,
) -> List[TiebaNote]:
"""
根据创作者用户名获取创作者所有帖子
Get all creator posts by creator username
Args:
user_name: 创作者用户名
crawl_interval: 爬取一次笔记的延迟单位（秒）
callback: 一次笔记爬取结束后的回调函数，是一个awaitable类型的函数
max_note_count: 帖子最大获取数量，如果为0则获取所有
creator_page_html_content: 创作者主页HTML内容
user_name: Creator username
crawl_interval: Crawl delay interval in seconds
callback: Callback function after one post crawl completes, an awaitable function
max_note_count: Maximum number of posts to retrieve, if 0 then get all
creator_page_html_content: Creator homepage HTML content
Returns:
"""
# 百度贴吧比较特殊一些，前10个帖子是直接展示在主页上的，要单独处理，通过API获取不到
# Baidu Tieba is special, the first 10 posts are directly displayed on the homepage and need special handling, cannot be obtained through API
result: List[TiebaNote] = []
if creator_page_html_content:
thread_id_list = (self._page_extractor.extract_tieba_thread_id_list_from_creator_page(creator_page_html_content))
media_platform/tieba/core.py
+61 -61
View File
@@ -79,9 +79,9 @@ class TieBaCrawler(AbstractCrawler):
)
async with async_playwright() as playwright:
# 根据配置选择启动模式
# Choose startup mode based on configuration
if config.ENABLE_CDP_MODE:
utils.logger.info("[BaiduTieBaCrawler] 使用CDP模式启动浏览器")
utils.logger.info("[BaiduTieBaCrawler] Launching browser in CDP mode")
self.browser_context = await self.launch_browser_with_cdp(
playwright,
playwright_proxy_format,
@@ -89,7 +89,7 @@ class TieBaCrawler(AbstractCrawler):
headless=config.CDP_HEADLESS,
)
else:
utils.logger.info("[BaiduTieBaCrawler] 使用标准模式启动浏览器")
utils.logger.info("[BaiduTieBaCrawler] Launching browser in standard mode")
# Launch a browser context.
chromium = playwright.chromium
self.browser_context = await self.launch_browser(
@@ -99,12 +99,12 @@ class TieBaCrawler(AbstractCrawler):
headless=config.HEADLESS,
)
# 注入反检测脚本 - 针对百度的特殊检测
# Inject anti-detection scripts - for Baidu's special detection
await self._inject_anti_detection_scripts()
self.context_page = await self.browser_context.new_page()
# 先访问百度首页,再点击贴吧链接,避免触发安全验证
# First visit Baidu homepage, then click Tieba link to avoid triggering security verification
await self._navigate_to_tieba_via_baidu()
# Create a client to interact with the baidutieba website.
@@ -399,29 +399,29 @@ class TieBaCrawler(AbstractCrawler):
async def _navigate_to_tieba_via_baidu(self):
"""
模拟真实用户访问路径:
1. 先访问百度首页 (https://www.baidu.com/)
2. 等待页面加载
3. 点击顶部导航栏的"贴吧"链接
4. 跳转到贴吧首页
Simulate real user access path:
1. First visit Baidu homepage (https://www.baidu.com/)
2. Wait for page to load
3. Click "Tieba" link in top navigation bar
4. Jump to Tieba homepage
这样做可以避免触发百度的安全验证
This avoids triggering Baidu's security verification
"""
utils.logger.info("[TieBaCrawler] 模拟真实用户访问路径...")
utils.logger.info("[TieBaCrawler] Simulating real user access path...")
try:
# Step 1: 访问百度首页
utils.logger.info("[TieBaCrawler] Step 1: 访问百度首页 https://www.baidu.com/")
# Step 1: Visit Baidu homepage
utils.logger.info("[TieBaCrawler] Step 1: Visiting Baidu homepage https://www.baidu.com/")
await self.context_page.goto("https://www.baidu.com/", wait_until="domcontentloaded")
# Step 2: 等待页面加载,使用配置文件中的延时设置
utils.logger.info(f"[TieBaCrawler] Step 2: 等待 {config.CRAWLER_MAX_SLEEP_SEC}秒 模拟用户浏览...")
# Step 2: Wait for page loading, using delay setting from config file
utils.logger.info(f"[TieBaCrawler] Step 2: Waiting {config.CRAWLER_MAX_SLEEP_SEC} seconds to simulate user browsing...")
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
# Step 3: 查找并点击"贴吧"链接
utils.logger.info("[TieBaCrawler] Step 3: 查找并点击'贴吧'链接...")
# Step 3: Find and click "Tieba" link
utils.logger.info("[TieBaCrawler] Step 3: Finding and clicking 'Tieba' link...")
# 尝试多种选择器,确保能找到贴吧链接
# Try multiple selectors to ensure finding the Tieba link
tieba_selectors = [
'a[href="http://tieba.baidu.com/"]',
'a[href="https://tieba.baidu.com/"]',
@@ -434,74 +434,74 @@ class TieBaCrawler(AbstractCrawler):
try:
tieba_link = await self.context_page.wait_for_selector(selector, timeout=5000)
if tieba_link:
utils.logger.info(f"[TieBaCrawler] 找到贴吧链接 (selector: {selector})")
utils.logger.info(f"[TieBaCrawler] Found Tieba link (selector: {selector})")
break
except Exception:
continue
if not tieba_link:
utils.logger.warning("[TieBaCrawler] 未找到贴吧链接,直接访问贴吧首页")
utils.logger.warning("[TieBaCrawler] Tieba link not found, directly accessing Tieba homepage")
await self.context_page.goto(self.index_url, wait_until="domcontentloaded")
return
# Step 4: 点击贴吧链接 (检查是否会打开新标签页)
utils.logger.info("[TieBaCrawler] Step 4: 点击贴吧链接...")
# Step 4: Click Tieba link (check if it will open in a new tab)
utils.logger.info("[TieBaCrawler] Step 4: Clicking Tieba link...")
# 检查链接的target属性
# Check link's target attribute
target_attr = await tieba_link.get_attribute("target")
utils.logger.info(f"[TieBaCrawler] 链接target属性: {target_attr}")
utils.logger.info(f"[TieBaCrawler] Link target attribute: {target_attr}")
if target_attr == "_blank":
# 如果是新标签页,需要等待新页面并切换
utils.logger.info("[TieBaCrawler] 链接会在新标签页打开,等待新页面...")
# If it's a new tab, need to wait for new page and switch
utils.logger.info("[TieBaCrawler] Link will open in new tab, waiting for new page...")
async with self.browser_context.expect_page() as new_page_info:
await tieba_link.click()
# 获取新打开的页面
# Get newly opened page
new_page = await new_page_info.value
await new_page.wait_for_load_state("domcontentloaded")
# 关闭旧的百度首页
# Close old Baidu homepage
await self.context_page.close()
# 切换到新的贴吧页面
# Switch to new Tieba page
self.context_page = new_page
utils.logger.info("[TieBaCrawler] ✅ 已切换到新标签页 (贴吧页面)")
utils.logger.info("[TieBaCrawler] Successfully switched to new tab (Tieba page)")
else:
# 如果是同一标签页跳转,正常等待导航
utils.logger.info("[TieBaCrawler] 链接在当前标签页跳转...")
# If it's same tab navigation, wait for navigation normally
utils.logger.info("[TieBaCrawler] Link navigates in current tab...")
async with self.context_page.expect_navigation(wait_until="domcontentloaded"):
await tieba_link.click()
# Step 5: 等待页面稳定,使用配置文件中的延时设置
utils.logger.info(f"[TieBaCrawler] Step 5: 页面加载完成,等待 {config.CRAWLER_MAX_SLEEP_SEC}秒...")
# Step 5: Wait for page to stabilize, using delay setting from config file
utils.logger.info(f"[TieBaCrawler] Step 5: Page loaded, waiting {config.CRAWLER_MAX_SLEEP_SEC} seconds...")
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
current_url = self.context_page.url
utils.logger.info(f"[TieBaCrawler] ✅ 成功通过百度首页进入贴吧! 当前URL: {current_url}")
utils.logger.info(f"[TieBaCrawler] Successfully entered Tieba via Baidu homepage! Current URL: {current_url}")
except Exception as e:
utils.logger.error(f"[TieBaCrawler] 通过百度首页访问贴吧失败: {e}")
utils.logger.info("[TieBaCrawler] 回退:直接访问贴吧首页")
utils.logger.error(f"[TieBaCrawler] Failed to access Tieba via Baidu homepage: {e}")
utils.logger.info("[TieBaCrawler] Fallback: directly accessing Tieba homepage")
await self.context_page.goto(self.index_url, wait_until="domcontentloaded")
async def _inject_anti_detection_scripts(self):
"""
注入反检测JavaScript脚本
针对百度贴吧的特殊检测机制
Inject anti-detection JavaScript scripts
For Baidu Tieba's special detection mechanism
"""
utils.logger.info("[TieBaCrawler] Injecting anti-detection scripts...")
# 轻量级反检测脚本,只覆盖关键检测点
# Lightweight anti-detection script, only covering key detection points
anti_detection_js = """
// 覆盖 navigator.webdriver
// Override navigator.webdriver
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined,
configurable: true
});
// 覆盖 window.navigator.chrome
// Override window.navigator.chrome
if (!window.navigator.chrome) {
window.navigator.chrome = {
runtime: {},
@@ -511,7 +511,7 @@ class TieBaCrawler(AbstractCrawler):
};
}
// 覆盖 Permissions API
// Override Permissions API
const originalQuery = window.navigator.permissions.query;
window.navigator.permissions.query = (parameters) => (
parameters.name === 'notifications' ?
@@ -519,19 +519,19 @@ class TieBaCrawler(AbstractCrawler):
originalQuery(parameters)
);
// 覆盖 plugins 长度(让它看起来有插件)
// Override plugins length (make it look like there are plugins)
Object.defineProperty(navigator, 'plugins', {
get: () => [1, 2, 3, 4, 5],
configurable: true
});
// 覆盖 languages
// Override languages
Object.defineProperty(navigator, 'languages', {
get: () => ['zh-CN', 'zh', 'en'],
configurable: true
});
// 移除 window.cdc_ 等 ChromeDriver 残留
// Remove window.cdc_ and other ChromeDriver remnants
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;
@@ -548,21 +548,21 @@ class TieBaCrawler(AbstractCrawler):
"""
Create tieba client with real browser User-Agent and complete headers
Args:
httpx_proxy: HTTP代理
ip_pool: IP代理池
httpx_proxy: HTTP proxy
ip_pool: IP proxy pool
Returns:
BaiduTieBaClient实例
BaiduTieBaClient instance
"""
utils.logger.info("[TieBaCrawler.create_tieba_client] Begin create tieba API client...")
# 从真实浏览器提取User-Agent,避免被检测
# Extract User-Agent from real browser to avoid detection
user_agent = await self.context_page.evaluate("() => navigator.userAgent")
utils.logger.info(f"[TieBaCrawler.create_tieba_client] Extracted User-Agent from browser: {user_agent}")
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
# 构建完整的浏览器请求头,模拟真实浏览器行为
# Build complete browser request headers, simulating real browser behavior
tieba_client = BaiduTieBaClient(
timeout=10,
ip_pool=ip_pool,
@@ -572,7 +572,7 @@ class TieBaCrawler(AbstractCrawler):
"Accept-Language": "zh-CN,zh;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
"User-Agent": user_agent, # 使用真实浏览器的UA
"User-Agent": user_agent, # Use real browser UA
"Cookie": cookie_str,
"Host": "tieba.baidu.com",
"Referer": "https://tieba.baidu.com/",
@@ -585,7 +585,7 @@ class TieBaCrawler(AbstractCrawler):
"sec-ch-ua-mobile": "?0",
"sec-ch-ua-platform": '"macOS"',
},
playwright_page=self.context_page, # 传入playwright页面对象
playwright_page=self.context_page, # Pass in playwright page object
)
return tieba_client
@@ -623,7 +623,7 @@ class TieBaCrawler(AbstractCrawler):
proxy=playwright_proxy, # type: ignore
viewport={"width": 1920, "height": 1080},
user_agent=user_agent,
channel="chrome", # 使用系统的Chrome稳定版
channel="chrome", # Use system's stable Chrome version
)
return browser_context
else:
@@ -641,7 +641,7 @@ class TieBaCrawler(AbstractCrawler):
headless: bool = True,
) -> BrowserContext:
"""
使用CDP模式启动浏览器
Launch browser using CDP mode
"""
try:
self.cdp_manager = CDPBrowserManager()
@@ -652,15 +652,15 @@ class TieBaCrawler(AbstractCrawler):
headless=headless,
)
# 显示浏览器信息
# Display browser information
browser_info = await self.cdp_manager.get_browser_info()
utils.logger.info(f"[TieBaCrawler] CDP浏览器信息: {browser_info}")
utils.logger.info(f"[TieBaCrawler] CDP browser info: {browser_info}")
return browser_context
except Exception as e:
utils.logger.error(f"[TieBaCrawler] CDP模式启动失败，回退到标准模式: {e}")
# 回退到标准模式
utils.logger.error(f"[TieBaCrawler] CDP mode launch failed, falling back to standard mode: {e}")
# Fall back to standard mode
chromium = playwright.chromium
return await self.launch_browser(
chromium, playwright_proxy, user_agent, headless
@@ -672,7 +672,7 @@ class TieBaCrawler(AbstractCrawler):
Returns:
"""
# 如果使用CDP模式，需要特殊处理
# If using CDP mode, need special handling
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
media_platform/tieba/field.py
+5 -5
View File
@@ -23,16 +23,16 @@ from enum import Enum
class SearchSortType(Enum):
"""search sort type"""
# 按时间倒序
# Sort by time in descending order
TIME_DESC = "1"
# 按时间顺序
# Sort by time in ascending order
TIME_ASC = "0"
# 按相关性顺序
# Sort by relevance
RELEVANCE_ORDER = "2"
class SearchNoteType(Enum):
# 只看主题贴
# Only view main posts
MAIN_THREAD = "1"
# 混合模式（帖子+回复）
# Mixed mode (posts + replies)
FIXED_THREAD = "0"
media_platform/tieba/help.py
+54 -48
View File
@@ -42,12 +42,12 @@ class TieBaExtractor:
@staticmethod
def extract_search_note_list(page_content: str) -> List[TiebaNote]:
"""
提取贴吧帖子列表，这里提取的关键词搜索结果页的数据，还缺少帖子的回复数和回复页等数据
Extract Tieba post list from keyword search result pages, still missing reply count and reply page data
Args:
page_content: 页面内容的HTML字符串
page_content: HTML string of page content
Returns:
包含帖子信息的字典列表
List of Tieba post objects
"""
xpath_selector = "//div[@class='s_post']"
post_list = Selector(text=page_content).xpath(xpath_selector)
@@ -71,12 +71,12 @@ class TieBaExtractor:
def extract_tieba_note_list(self, page_content: str) -> List[TiebaNote]:
"""
提取贴吧帖子列表
Extract Tieba post list from Tieba page
Args:
page_content:
page_content: HTML string of page content
Returns:
List of Tieba post objects
"""
page_content = page_content.replace('<!--', "")
content_selector = Selector(text=page_content)
@@ -106,21 +106,21 @@ class TieBaExtractor:
def extract_note_detail(self, page_content: str) -> TiebaNote:
"""
提取贴吧帖子详情
Extract Tieba post details from post detail page
Args:
page_content:
page_content: HTML string of page content
Returns:
Tieba post detail object
"""
content_selector = Selector(text=page_content)
first_floor_selector = content_selector.xpath("//div[@class='p_postlist'][1]")
only_view_author_link = content_selector.xpath("//*[@id='lzonly_cntn']/@href").get(default='').strip()
note_id = only_view_author_link.split("?")[0].split("/")[-1]
# 帖子回复数、回复页数
# Post reply count and reply page count
thread_num_infos = content_selector.xpath(
"//div[@id='thread_theme_5']//li[@class='l_reply_num']//span[@class='red']")
# IP地理位置、发表时间
# IP location and publish time
other_info_content = content_selector.xpath(".//div[@class='post-tail-wrap']").get(default="").strip()
ip_location, publish_time = self.extract_ip_and_pub_time(other_info_content)
note = TiebaNote(note_id=note_id, title=content_selector.xpath("//title/text()").get(default='').strip(),
@@ -138,18 +138,18 @@ class TieBaExtractor:
publish_time=publish_time,
total_replay_num=thread_num_infos[0].xpath("./text()").get(default='').strip(),
total_replay_page=thread_num_infos[1].xpath("./text()").get(default='').strip(), )
note.title = note.title.replace(f"【{note.tieba_name}】_百度贴吧", "")
note.title = note.title.replace(f"【{note.tieba_name}】_Baidu Tieba", "")
return note
def extract_tieba_note_parment_comments(self, page_content: str, note_id: str) -> List[TiebaComment]:
"""
提取贴吧帖子一级评论
Extract Tieba post first-level comments from comment page
Args:
page_content:
note_id:
page_content: HTML string of page content
note_id: Post ID
Returns:
List of first-level comment objects
"""
xpath_selector = "//div[@class='l_post l_post_bright j_l_post clearfix ']"
comment_list = Selector(text=page_content).xpath(xpath_selector)
@@ -180,13 +180,13 @@ class TieBaExtractor:
def extract_tieba_note_sub_comments(self, page_content: str, parent_comment: TiebaComment) -> List[TiebaComment]:
"""
提取贴吧帖子二级评论
Extract Tieba post second-level comments from sub-comment page
Args:
page_content:
parent_comment:
page_content: HTML string of page content
parent_comment: Parent comment object
Returns:
List of second-level comment objects
"""
selector = Selector(page_content)
comments = []
@@ -215,12 +215,12 @@ class TieBaExtractor:
def extract_creator_info(self, html_content: str) -> TiebaCreator:
"""
提取贴吧创作者信息
Extract Tieba creator information from creator homepage
Args:
html_content:
html_content: HTML string of creator homepage
Returns:
Tieba creator object
"""
selector = Selector(text=html_content)
user_link_selector = selector.xpath("//p[@class='space']/a")
@@ -251,12 +251,12 @@ class TieBaExtractor:
html_content: str
) -> List[str]:
"""
提取贴吧创作者主页的帖子列表
Extract post ID list from Tieba creator's homepage
Args:
html_content:
html_content: HTML string of creator homepage
Returns:
List of post IDs
"""
selector = Selector(text=html_content)
thread_id_list = []
@@ -271,12 +271,12 @@ class TieBaExtractor:
def extract_ip_and_pub_time(self, html_content: str) -> Tuple[str, str]:
"""
提取IP位置和发布时间
Extract IP location and publish time from HTML content
Args:
html_content:
html_content: HTML string
Returns:
Tuple of (IP location, publish time)
"""
pattern_pub_time = re.compile(r'<span class="tail-info">(\d{4}-\d{2}-\d{2} \d{2}:\d{2})</span>')
time_match = pattern_pub_time.search(html_content)
@@ -286,12 +286,12 @@ class TieBaExtractor:
@staticmethod
def extract_ip(html_content: str) -> str:
"""
提取IP
Extract IP location from HTML content
Args:
html_content:
html_content: HTML string
Returns:
IP location string
"""
pattern_ip = re.compile(r'IP属地:(\S+)</span>')
ip_match = pattern_ip.search(html_content)
@@ -301,28 +301,28 @@ class TieBaExtractor:
@staticmethod
def extract_gender(html_content: str) -> str:
"""
提取性别
Extract gender from HTML content
Args:
html_content:
html_content: HTML string
Returns:
Gender string ('Male', 'Female', or 'Unknown')
"""
if GENDER_MALE in html_content:
return '男'
return 'Male'
elif GENDER_FEMALE in html_content:
return '女'
return '未知'
return 'Female'
return 'Unknown'
@staticmethod
def extract_follow_and_fans(selectors: List[Selector]) -> Tuple[str, str]:
"""
提取关注数和粉丝数
Extract follow count and fan count from selectors
Args:
selectors:
selectors: List of selector objects
Returns:
Tuple of (follow count, fan count)
"""
pattern = re.compile(r'<span class="concern_num">\(<a[^>]*>(\d+)</a>\)</span>')
follow_match = pattern.findall(selectors[0].get())
@@ -334,9 +334,15 @@ class TieBaExtractor:
@staticmethod
def extract_registration_duration(html_content: str) -> str:
"""
"<span>吧龄:1.9年</span>"
Returns: 1.9年
Extract Tieba age from HTML content
Example: "<span>吧龄:1.9年</span>"
Returns: "1.9年"
Args:
html_content: HTML string
Returns:
Tieba age string
"""
pattern = re.compile(r'<span>吧龄:(\S+)</span>')
match = pattern.search(html_content)
@@ -345,22 +351,22 @@ class TieBaExtractor:
@staticmethod
def extract_data_field_value(selector: Selector) -> Dict:
"""
提取data-field的值
Extract data-field value from selector
Args:
selector:
selector: Selector object
Returns:
Dictionary containing data-field value
"""
data_field_value = selector.xpath("./@data-field").get(default='').strip()
if not data_field_value or data_field_value == "{}":
return {}
try:
# 先使用 html.unescape 处理转义字符 再json.loads 将 JSON 字符串转换为 Python 字典
# First use html.unescape to handle escape characters, then json.loads to convert JSON string to Python dictionary
unescaped_json_str = html.unescape(data_field_value)
data_field_dict_value = json.loads(unescaped_json_str)
except Exception as ex:
print(f"extract_data_field_value，错误信息：{ex}, 尝试使用其他方式解析")
print(f"extract_data_field_value, error: {ex}, trying alternative parsing method")
data_field_dict_value = {}
return data_field_dict_value
media_platform/tieba/login.py
+1 -1
View File
@@ -50,7 +50,7 @@ class BaiduTieBaLogin(AbstractLogin):
@retry(stop=stop_after_attempt(600), wait=wait_fixed(1), retry=retry_if_result(lambda value: value is False))
async def check_login_state(self) -> bool:
"""
轮训检查登录状态是否成功，成功返回True否则返回False
Poll to check if login status is successful, return True if successful, otherwise return False
Returns:
media_platform/weibo/client.py
+37 -37
View File
@@ -20,7 +20,7 @@
# -*- coding: utf-8 -*-
# @Author : [email protected]
# @Time : 2023/12/23 15:40
# @Desc : 微博爬虫 API 请求 client
# @Desc : Weibo crawler API request client
import asyncio
import copy
@@ -49,7 +49,7 @@ class WeiboClient(ProxyRefreshMixin):
def __init__(
self,
timeout=60, # 若开启爬取媒体选项，weibo 的图片需要更久的超时时间
timeout=60, # If media crawling is enabled, Weibo images need a longer timeout
proxy=None,
*,
headers: Dict[str, str],
@@ -64,12 +64,12 @@ class WeiboClient(ProxyRefreshMixin):
self.playwright_page = playwright_page
self.cookie_dict = cookie_dict
self._image_agent_host = "https://i1.wp.com/"
# 初始化代理池（来自 ProxyRefreshMixin）
# Initialize proxy pool (from ProxyRefreshMixin)
self.init_proxy_pool(proxy_ip_pool)
@retry(stop=stop_after_attempt(5), wait=wait_fixed(3))
async def request(self, method, url, **kwargs) -> Union[Response, Dict]:
# 每次请求前检测代理是否过期
# Check if proxy is expired before each request
await self._refresh_proxy_if_expired()
enable_return_response = kwargs.pop("return_response", False)
@@ -82,7 +82,7 @@ class WeiboClient(ProxyRefreshMixin):
try:
data: Dict = response.json()
except json.decoder.JSONDecodeError:
# issue: #771 搜索接口会报错432， 多次重试 + 更新 h5 cookies
# issue: #771 Search API returns error 432, retry multiple times + update h5 cookies
utils.logger.error(f"[WeiboClient.request] request {method}:{url} err code: {response.status_code} res:{response.text}")
await self.playwright_page.goto(self._host)
await asyncio.sleep(2)
@@ -156,9 +156,9 @@ class WeiboClient(ProxyRefreshMixin):
) -> Dict:
"""
search note by keyword
:param keyword: 微博搜搜的关键词
:param page: 分页参数 -当前页码
:param search_type: 搜索的类型，见 weibo/filed.py 中的枚举SearchType
:param keyword: Search keyword for Weibo
:param page: Pagination parameter - current page number
:param search_type: Search type, see SearchType enum in weibo/field.py
:return:
"""
uri = "/api/container/getIndex"
@@ -172,9 +172,9 @@ class WeiboClient(ProxyRefreshMixin):
async def get_note_comments(self, mid_id: str, max_id: int, max_id_type: int = 0) -> Dict:
"""get notes comments
:param mid_id: 微博ID
:param max_id: 分页参数ID
:param max_id_type: 分页参数ID类型
:param mid_id: Weibo ID
:param max_id: Pagination parameter ID
:param max_id_type: Pagination parameter ID type
:return:
"""
uri = "/comments/hotflow"
@@ -218,7 +218,7 @@ class WeiboClient(ProxyRefreshMixin):
is_end = max_id == 0
if len(result) + len(comment_list) > max_count:
comment_list = comment_list[:max_count - len(result)]
if callback: # 如果有回调函数，就执行回调函数
if callback: # If callback function exists, execute it
await callback(note_id, comment_list)
await asyncio.sleep(crawl_interval)
result.extend(comment_list)
@@ -233,7 +233,7 @@ class WeiboClient(ProxyRefreshMixin):
callback: Optional[Callable] = None,
) -> List[Dict]:
"""
获取评论的所有子评论
Get all sub-comments of comments
Args:
note_id:
comment_list:
@@ -256,7 +256,7 @@ class WeiboClient(ProxyRefreshMixin):
async def get_note_info_by_id(self, note_id: str) -> Dict:
"""
根据帖子ID获取详情
Get note details by note ID
:param note_id:
:return:
"""
@@ -273,22 +273,22 @@ class WeiboClient(ProxyRefreshMixin):
note_item = {"mblog": note_detail}
return note_item
else:
utils.logger.info(f"[WeiboClient.get_note_info_by_id] 未找到$render_data的值")
utils.logger.info(f"[WeiboClient.get_note_info_by_id] $render_data value not found")
return dict()
async def get_note_image(self, image_url: str) -> bytes:
image_url = image_url[8:] # 去掉 https://
image_url = image_url[8:] # Remove https://
sub_url = image_url.split("/")
image_url = ""
for i in range(len(sub_url)):
if i == 1:
image_url += "large/" # 都获取高清大图
image_url += "large/" # Get high-resolution images
elif i == len(sub_url) - 1:
image_url += sub_url[i]
else:
image_url += sub_url[i] + "/"
# 微博图床对外存在防盗链，所以需要代理访问
# 由于微博图片是通过 i1.wp.com 来访问的，所以需要拼接一下
# Weibo image hosting has anti-hotlinking, so proxy access is needed
# Since Weibo images are accessed through i1.wp.com, we need to concatenate the URL
final_uri = (f"{self._image_agent_host}"
f"{image_url}")
async with httpx.AsyncClient(proxy=self.proxy) as client:
@@ -301,18 +301,18 @@ class WeiboClient(ProxyRefreshMixin):
else:
return response.content
except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # 保留原始异常类型名称，以便开发者调试
utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # Keep original exception type name for developer debugging
return None
async def get_creator_container_info(self, creator_id: str) -> Dict:
"""
获取用户的容器ID, 容器信息代表着真实请求的API路径
fid_container_id：用户的微博详情API的容器ID
lfid_container_id：用户的微博列表API的容器ID
Get user's container ID, container information represents the real API request path
fid_container_id: Container ID for user's Weibo detail API
lfid_container_id: Container ID for user's Weibo list API
Args:
creator_id:
creator_id: User ID
Returns: {
Returns: Dictionary with container IDs
"""
response = await self.get(f"/u/{creator_id}", return_response=True)
@@ -324,7 +324,7 @@ class WeiboClient(ProxyRefreshMixin):
async def get_creator_info_by_id(self, creator_id: str) -> Dict:
"""
根据用户ID获取用户详情
Get user details by user ID
Args:
creator_id:
@@ -349,11 +349,11 @@ class WeiboClient(ProxyRefreshMixin):
since_id: str = "0",
) -> Dict:
"""
获取博主的笔记
Get creator's notes
Args:
creator: 博主ID
container_id: 容器ID
since_id: 上一页最后一条笔记的ID
creator: Creator ID
container_id: Container ID
since_id: ID of the last note from previous page
Returns:
"""
@@ -376,14 +376,14 @@ class WeiboClient(ProxyRefreshMixin):
callback: Optional[Callable] = None,
) -> List[Dict]:
"""
获取指定用户下的所有发过的帖子，该方法会一直查找一个用户下的所有帖子信息
Get all posts published by a specified user, this method will continuously fetch all posts from a user
Args:
creator_id:
container_id:
crawl_interval:
callback:
creator_id: Creator user ID
container_id: Container ID for the user
crawl_interval: Interval between requests in seconds
callback: Optional callback function to process notes
Returns:
Returns: List of all notes
"""
result = []
@@ -393,7 +393,7 @@ class WeiboClient(ProxyRefreshMixin):
while notes_has_more:
notes_res = await self.get_notes_by_creator(creator_id, container_id, since_id)
if not notes_res:
utils.logger.error(f"[WeiboClient.get_notes_by_creator] The current creator may have been banned by xhs, so they cannot access the data.")
utils.logger.error(f"[WeiboClient.get_notes_by_creator] The current creator may have been banned by Weibo, so they cannot access the data.")
break
since_id = notes_res.get("cardlistInfo", {}).get("since_id", "0")
if "cards" not in notes_res:
media_platform/weibo/core.py
+28 -28
View File
@@ -20,7 +20,7 @@
# -*- coding: utf-8 -*-
# @Author : [email protected]
# @Time : 2023/12/23 15:41
# @Desc : 微博爬虫主流程代码
# @Desc : Weibo crawler main workflow code
import asyncio
import os
@@ -63,7 +63,7 @@ class WeiboCrawler(AbstractCrawler):
self.user_agent = utils.get_user_agent()
self.mobile_user_agent = utils.get_mobile_user_agent()
self.cdp_manager = None
self.ip_proxy_pool = None # 代理IP池，用于代理自动刷新
self.ip_proxy_pool = None # Proxy IP pool for automatic proxy refresh
async def start(self):
playwright_proxy_format, httpx_proxy_format = None, None
@@ -73,9 +73,9 @@ class WeiboCrawler(AbstractCrawler):
playwright_proxy_format, httpx_proxy_format = utils.format_proxy_info(ip_proxy_info)
async with async_playwright() as playwright:
# 根据配置选择启动模式
# Select launch mode based on configuration
if config.ENABLE_CDP_MODE:
utils.logger.info("[WeiboCrawler] 使用CDP模式启动浏览器")
utils.logger.info("[WeiboCrawler] Launching browser with CDP mode")
self.browser_context = await self.launch_browser_with_cdp(
playwright,
playwright_proxy_format,
@@ -83,7 +83,7 @@ class WeiboCrawler(AbstractCrawler):
headless=config.CDP_HEADLESS,
)
else:
utils.logger.info("[WeiboCrawler] 使用标准模式启动浏览器")
utils.logger.info("[WeiboCrawler] Launching browser with standard mode")
# Launch a browser context.
chromium = playwright.chromium
self.browser_context = await self.launch_browser(chromium, None, self.mobile_user_agent, headless=config.HEADLESS)
@@ -109,11 +109,11 @@ class WeiboCrawler(AbstractCrawler):
)
await login_obj.begin()
# 登录成功后重定向到手机端的网站，再更新手机端登录成功的cookie
# After successful login, redirect to mobile website and update mobile cookies
utils.logger.info("[WeiboCrawler.start] redirect weibo mobile homepage and update cookies on mobile platform")
await self.context_page.goto(self.mobile_index_url)
await asyncio.sleep(3)
# 只获取移动端的 cookies，避免 PC 端和移动端 cookies 混淆
# Only get mobile cookies to avoid confusion between PC and mobile cookies
await self.wb_client.update_cookies(
browser_context=self.browser_context,
urls=[self.mobile_index_url]
@@ -170,7 +170,7 @@ class WeiboCrawler(AbstractCrawler):
search_res = await self.wb_client.get_note_by_keyword(keyword=keyword, page=page, search_type=search_type)
note_id_list: List[str] = []
note_list = filter_search_result_card(search_res.get("cards"))
# 如果开启了全文获取功能，则批量获取帖子全文
# If full text fetching is enabled, batch get full text of posts
note_list = await self.batch_get_notes_full_text(note_list)
for note_item in note_list:
if note_item:
@@ -315,9 +315,9 @@ class WeiboCrawler(AbstractCrawler):
raise DataFetchError("Get creator info error")
await weibo_store.save_creator(user_id, user_info=createor_info)
# 创建一个包装 callback，在保存数据前获取全文
# Create a wrapper callback to get full text before saving data
async def save_notes_with_full_text(note_list: List[Dict]):
# 如果开启了全文获取功能，先批量获取全文
# If full text fetching is enabled, batch get full text first
updated_note_list = await self.batch_get_notes_full_text(note_list)
await weibo_store.batch_update_weibo_notes(updated_note_list)
@@ -350,7 +350,7 @@ class WeiboCrawler(AbstractCrawler):
},
playwright_page=self.context_page,
cookie_dict=cookie_dict,
proxy_ip_pool=self.ip_proxy_pool, # 传递代理池用于自动刷新
proxy_ip_pool=self.ip_proxy_pool, # Pass proxy pool for automatic refresh
)
return weibo_client_obj
@@ -375,7 +375,7 @@ class WeiboCrawler(AbstractCrawler):
"height": 1080
},
user_agent=user_agent,
channel="chrome", # 使用系统的Chrome稳定版
channel="chrome", # Use system's Chrome stable version
)
return browser_context
else:
@@ -391,7 +391,7 @@ class WeiboCrawler(AbstractCrawler):
headless: bool = True,
) -> BrowserContext:
"""
使用CDP模式启动浏览器
Launch browser with CDP mode
"""
try:
self.cdp_manager = CDPBrowserManager()
@@ -402,24 +402,24 @@ class WeiboCrawler(AbstractCrawler):
headless=headless,
)
# 显示浏览器信息
# Display browser information
browser_info = await self.cdp_manager.get_browser_info()
utils.logger.info(f"[WeiboCrawler] CDP浏览器信息: {browser_info}")
utils.logger.info(f"[WeiboCrawler] CDP browser info: {browser_info}")
return browser_context
except Exception as e:
utils.logger.error(f"[WeiboCrawler] CDP模式启动失败，回退到标准模式: {e}")
# 回退到标准模式
utils.logger.error(f"[WeiboCrawler] CDP mode startup failed, falling back to standard mode: {e}")
# Fallback to standard mode
chromium = playwright.chromium
return await self.launch_browser(chromium, playwright_proxy, user_agent, headless)
async def get_note_full_text(self, note_item: Dict) -> Dict:
"""
获取帖子全文内容
如果帖子内容被截断（isLongText=True），则请求详情接口获取完整内容
:param note_item: 帖子数据，包含 mblog 字段
:return: 更新后的帖子数据
Get full text content of a post
If the post content is truncated (isLongText=True), request the detail API to get complete content
:param note_item: Post data, contains mblog field
:return: Updated post data
"""
if not config.ENABLE_WEIBO_FULL_TEXT:
return note_item
@@ -428,7 +428,7 @@ class WeiboCrawler(AbstractCrawler):
if not mblog:
return note_item
# 检查是否是长文本
# Check if it's a long text
is_long_text = mblog.get("isLongText", False)
if not is_long_text:
return note_item
@@ -441,11 +441,11 @@ class WeiboCrawler(AbstractCrawler):
utils.logger.info(f"[WeiboCrawler.get_note_full_text] Fetching full text for note: {note_id}")
full_note = await self.wb_client.get_note_info_by_id(note_id)
if full_note and full_note.get("mblog"):
# 用完整内容替换原始内容
# Replace original content with complete content
note_item["mblog"] = full_note["mblog"]
utils.logger.info(f"[WeiboCrawler.get_note_full_text] Successfully fetched full text for note: {note_id}")
# 请求后休眠，避免风控
# Sleep after request to avoid rate limiting
await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC)
except DataFetchError as ex:
utils.logger.error(f"[WeiboCrawler.get_note_full_text] Failed to fetch full text for note {note_id}: {ex}")
@@ -456,9 +456,9 @@ class WeiboCrawler(AbstractCrawler):
async def batch_get_notes_full_text(self, note_list: List[Dict]) -> List[Dict]:
"""
批量获取帖子全文内容
:param note_list: 帖子列表
:return: 更新后的帖子列表
Batch get full text content of posts
:param note_list: List of posts
:return: Updated list of posts
"""
if not config.ENABLE_WEIBO_FULL_TEXT:
return note_list
@@ -471,7 +471,7 @@ class WeiboCrawler(AbstractCrawler):
async def close(self):
"""Close browser context"""
# 如果使用CDP模式，需要特殊处理
# Special handling if using CDP mode
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
media_platform/weibo/field.py
+4 -4
View File
@@ -26,14 +26,14 @@ from enum import Enum
class SearchType(Enum):
# 综合
# Comprehensive
DEFAULT = "1"
# 实时
# Real-time
REAL_TIME = "61"
# 热门
# Popular
POPULAR = "60"
# 视频
# Video
VIDEO = "64"
media_platform/weibo/help.py
+3 -3
View File
@@ -28,9 +28,9 @@ from typing import Dict, List
def filter_search_result_card(card_list: List[Dict]) -> List[Dict]:
"""
过滤微博搜索的结果，只保留card_type为9类型的数据
:param card_list:
:return:
Filter Weibo search results, only keep data with card_type of 9
:param card_list: List of card items from search results
:return: Filtered list of note items
"""
note_list: List[Dict] = []
for card_item in card_list:
media_platform/weibo/login.py
+1 -1
View File
@@ -21,7 +21,7 @@
# -*- coding: utf-8 -*-
# @Author : [email protected]
# @Time : 2023/12/23 15:42
# @Desc : 微博登录实现
# @Desc : Weibo login implementation
import asyncio
import functools
media_platform/xhs/client.py
+83 -83
View File
@@ -45,7 +45,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
def __init__(
self,
timeout=60, # 若开启爬取媒体选项，xhs 的长视频需要更久的超时时间
timeout=60, # If media crawling is enabled, Xiaohongshu long videos need longer timeout
proxy=None,
*,
headers: Dict[str, str],
@@ -58,30 +58,30 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
self.headers = headers
self._host = "https://edith.xiaohongshu.com"
self._domain = "https://www.xiaohongshu.com"
self.IP_ERROR_STR = "网络连接异常，请检查网络设置或重启试试"
self.IP_ERROR_STR = "Network connection error, please check network settings or restart"
self.IP_ERROR_CODE = 300012
self.NOTE_ABNORMAL_STR = "笔记状态异常，请稍后查看"
self.NOTE_ABNORMAL_STR = "Note status abnormal, please check later"
self.NOTE_ABNORMAL_CODE = -510001
self.playwright_page = playwright_page
self.cookie_dict = cookie_dict
self._extractor = XiaoHongShuExtractor()
# 初始化代理池（来自 ProxyRefreshMixin）
# Initialize proxy pool (from ProxyRefreshMixin)
self.init_proxy_pool(proxy_ip_pool)
async def _pre_headers(self, url: str, params: Optional[Dict] = None, payload: Optional[Dict] = None) -> Dict:
"""请求头参数签名（使用 playwright 注入方式）
"""Request header parameter signing (using playwright injection method)
Args:
url: 请求的URL
params: GET请求的参数
payload: POST请求的参数
url: Request URL
params: GET request parameters
payload: POST request parameters
Returns:
Dict: 请求头参数签名
Dict: Signed request header parameters
"""
a1_value = self.cookie_dict.get("a1", "")
# 确定请求数据、方法和 URI
# Determine request data, method and URI
if params is not None:
data = params
method = "GET"
@@ -91,7 +91,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
else:
raise ValueError("params or payload is required")
# 使用 playwright 注入方式生成签名
# Generate signature using playwright injection method
signs = await sign_with_playwright(
page=self.playwright_page,
uri=url,
@@ -112,16 +112,16 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
@retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
async def request(self, method, url, **kwargs) -> Union[str, Any]:
"""
封装httpx的公共请求方法，对请求响应做一些处理
Wrapper for httpx common request method, processes request response
Args:
method: 请求方法
url: 请求的URL
**kwargs: 其他请求参数，例如请求头、请求体等
method: Request method
url: Request URL
**kwargs: Other request parameters, such as headers, body, etc.
Returns:
"""
# 每次请求前检测代理是否过期
# Check if proxy is expired before each request
await self._refresh_proxy_if_expired()
# return response.text
@@ -133,7 +133,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
# someday someone maybe will bypass captcha
verify_type = response.headers["Verifytype"]
verify_uuid = response.headers["Verifyuuid"]
msg = f"出现验证码，请求失败，Verifytype: {verify_type}，Verifyuuid: {verify_uuid}, Response: {response}"
msg = f"CAPTCHA appeared, request failed, Verifytype: {verify_type}, Verifyuuid: {verify_uuid}, Response: {response}"
utils.logger.error(msg)
raise Exception(msg)
@@ -150,10 +150,10 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
async def get(self, uri: str, params: Optional[Dict] = None) -> Dict:
"""
GET请求，对请求头签名
GET request, signs request headers
Args:
uri: 请求路由
params: 请求参数
uri: Request route
params: Request parameters
Returns:
@@ -167,10 +167,10 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
async def post(self, uri: str, data: dict, **kwargs) -> Dict:
"""
POST请求，对请求头签名
POST request, signs request headers
Args:
uri: 请求路由
data: 请求体参数
uri: Request route
data: Request body parameters
Returns:
@@ -186,7 +186,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
)
async def get_note_media(self, url: str) -> Union[bytes, None]:
# 请求前检测代理是否过期
# Check if proxy is expired before request
await self._refresh_proxy_if_expired()
async with httpx.AsyncClient(proxy=self.proxy) as client:
@@ -205,12 +205,12 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
) as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
utils.logger.error(
f"[XiaoHongShuClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}"
) # 保留原始异常类型名称，以便开发者调试
) # Keep original exception type name for developer debugging
return None
async def pong(self) -> bool:
"""
用于检查登录态是否失效了
Check if login state is still valid
Returns:
"""
@@ -218,7 +218,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
utils.logger.info("[XiaoHongShuClient.pong] Begin to pong xhs...")
ping_flag = False
try:
note_card: Dict = await self.get_note_by_keyword(keyword="小红书")
note_card: Dict = await self.get_note_by_keyword(keyword="Xiaohongshu")
if note_card.get("items"):
ping_flag = True
except Exception as e:
@@ -230,9 +230,9 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
async def update_cookies(self, browser_context: BrowserContext):
"""
API客户端提供的更新cookies方法，一般情况下登录成功后会调用此方法
Update cookies method provided by API client, usually called after successful login
Args:
browser_context: 浏览器上下文对象
browser_context: Browser context object
Returns:
@@ -251,13 +251,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
note_type: SearchNoteType = SearchNoteType.ALL,
) -> Dict:
"""
根据关键词搜索笔记
Search notes by keyword
Args:
keyword: 关键词参数
page: 分页第几页
page_size: 分页数据长度
sort: 搜索结果排序指定
note_type: 搜索的笔记类型
keyword: Keyword parameter
page: Page number
page_size: Page data length
sort: Search result sorting specification
note_type: Type of note to search
Returns:
@@ -280,11 +280,11 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
xsec_token: str,
) -> Dict:
"""
获取笔记详情API
Get note detail API
Args:
note_id:笔记ID
xsec_source: 渠道来源
xsec_token: 搜索关键字之后返回的比较列表中返回的token
note_id: Note ID
xsec_source: Channel source
xsec_token: Token returned from search keyword result list
Returns:
@@ -304,7 +304,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
if res and res.get("items"):
res_dict: Dict = res["items"][0]["note_card"]
return res_dict
# 爬取频繁了可能会出现有的笔记能有结果有的没有
# When crawling frequently, some notes may have results while others don't
utils.logger.error(
f"[XiaoHongShuClient.get_note_by_id] get note id:{note_id} empty and res:{res}"
)
@@ -317,11 +317,11 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
cursor: str = "",
) -> Dict:
"""
获取一级评论的API
Get first-level comments API
Args:
note_id: 笔记ID
xsec_token: 验证token
cursor: 分页游标
note_id: Note ID
xsec_token: Verification token
cursor: Pagination cursor
Returns:
@@ -345,13 +345,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
cursor: str = "",
):
"""
获取指定父评论下的子评论的API
Get sub-comments under specified parent comment API
Args:
note_id: 子评论的帖子ID
root_comment_id: 根评论ID
xsec_token: 验证token
num: 分页数量
cursor: 分页游标
note_id: Post ID of sub-comments
root_comment_id: Root comment ID
xsec_token: Verification token
num: Pagination quantity
cursor: Pagination cursor
Returns:
@@ -377,13 +377,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
max_count: int = 10,
) -> List[Dict]:
"""
获取指定笔记下的所有一级评论，该方法会一直查找一个帖子下的所有评论信息
Get all first-level comments under specified note, this method will continuously find all comment information under a post
Args:
note_id: 笔记ID
xsec_token: 验证token
crawl_interval: 爬取一次笔记的延迟单位（秒）
callback: 一次笔记爬取结束后
max_count: 一次笔记爬取的最大评论数量
note_id: Note ID
xsec_token: Verification token
crawl_interval: Crawl delay per note (seconds)
callback: Callback after one note crawl ends
max_count: Maximum number of comments to crawl per note
Returns:
"""
@@ -425,12 +425,12 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
callback: Optional[Callable] = None,
) -> List[Dict]:
"""
获取指定一级评论下的所有二级评论, 该方法会一直查找一级评论下的所有二级评论信息
Get all second-level comments under specified first-level comments, this method will continuously find all second-level comment information under first-level comments
Args:
comments: 评论列表
xsec_token: 验证token
crawl_interval: 爬取一次评论的延迟单位（秒）
callback: 一次评论爬取结束后
comments: Comment list
xsec_token: Verification token
crawl_interval: Crawl delay per comment (seconds)
callback: Callback after one comment crawl ends
Returns:
@@ -487,18 +487,18 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
self, user_id: str, xsec_token: str = "", xsec_source: str = ""
) -> Dict:
"""
通过解析网页版的用户主页HTML，获取用户个人简要信息
PC端用户主页的网页存在window.__INITIAL_STATE__这个变量上的，解析它即可
Get user profile brief information by parsing user homepage HTML
The PC user homepage has window.__INITIAL_STATE__ variable, just parse it
Args:
user_id: 用户ID
xsec_token: 验证token (可选,如果URL中包含此参数则传入)
xsec_source: 渠道来源 (可选,如果URL中包含此参数则传入)
user_id: User ID
xsec_token: Verification token (optional, pass if included in URL)
xsec_source: Channel source (optional, pass if included in URL)
Returns:
Dict: 创作者信息
Dict: Creator information
"""
# 构建URI,如果有xsec参数则添加到URL中
# Build URI, add xsec parameters to URL if available
uri = f"/user/profile/{user_id}"
if xsec_token and xsec_source:
uri = f"{uri}?xsec_token={xsec_token}&xsec_source={xsec_source}"
@@ -517,13 +517,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
xsec_source: str = "pc_feed",
) -> Dict:
"""
获取博主的笔记
Get creator's notes
Args:
creator: 博主ID
cursor: 上一页最后一条笔记的ID
page_size: 分页数据长度
xsec_token: 验证token
xsec_source: 渠道来源
creator: Creator ID
cursor: Last note ID from previous page
page_size: Page data length
xsec_token: Verification token
xsec_source: Channel source
Returns:
@@ -547,13 +547,13 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
xsec_source: str = "pc_feed",
) -> List[Dict]:
"""
获取指定用户下的所有发过的帖子，该方法会一直查找一个用户下的所有帖子信息
Get all posts published by specified user, this method will continuously find all post information under a user
Args:
user_id: 用户ID
crawl_interval: 爬取一次的延迟单位（秒）
callback: 一次分页爬取结束后的更新回调函数
xsec_token: 验证token
xsec_source: 渠道来源
user_id: User ID
crawl_interval: Crawl delay (seconds)
callback: Update callback function after one pagination crawl ends
xsec_token: Verification token
xsec_source: Channel source
Returns:
@@ -602,9 +602,9 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
async def get_note_short_url(self, note_id: str) -> Dict:
"""
获取笔记的短链接
Get note short URL
Args:
note_id: 笔记ID
note_id: Note ID
Returns:
@@ -622,7 +622,7 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin):
enable_cookie: bool = False,
) -> Optional[Dict]:
"""
通过解析网页版的笔记详情页HTML，获取笔记详情, 该接口可能会出现失败的情况，这里尝试重试3次
Get note details by parsing note detail page HTML, this interface may fail, retry 3 times here
copy from https://github.com/ReaJason/xhs/blob/eb1c5a0213f6fbb592f0a2897ee552847c69ea2d/xhs/core.py#L217-L259
thanks for ReaJason
Args:
media_platform/xhs/core.py
+31 -37
View File
@@ -60,7 +60,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
# self.user_agent = utils.get_user_agent()
self.user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
self.cdp_manager = None
self.ip_proxy_pool = None # 代理IP池，用于代理自动刷新
self.ip_proxy_pool = None # Proxy IP pool for automatic proxy refresh
async def start(self) -> None:
playwright_proxy_format, httpx_proxy_format = None, None
@@ -70,9 +70,9 @@ class XiaoHongShuCrawler(AbstractCrawler):
playwright_proxy_format, httpx_proxy_format = utils.format_proxy_info(ip_proxy_info)
async with async_playwright() as playwright:
# 根据配置选择启动模式
# Choose launch mode based on configuration
if config.ENABLE_CDP_MODE:
utils.logger.info("[XiaoHongShuCrawler] 使用CDP模式启动浏览器")
utils.logger.info("[XiaoHongShuCrawler] Launching browser using CDP mode")
self.browser_context = await self.launch_browser_with_cdp(
playwright,
playwright_proxy_format,
@@ -80,7 +80,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
headless=config.CDP_HEADLESS,
)
else:
utils.logger.info("[XiaoHongShuCrawler] 使用标准模式启动浏览器")
utils.logger.info("[XiaoHongShuCrawler] Launching browser using standard mode")
# Launch a browser context.
chromium = playwright.chromium
self.browser_context = await self.launch_browser(
@@ -95,7 +95,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
self.context_page = await self.browser_context.new_page()
await self.context_page.goto(self.index_url)
# Create a client to interact with the xiaohongshu website.
# Create a client to interact with the Xiaohongshu website.
self.xhs_client = await self.create_xhs_client(httpx_proxy_format)
if not await self.xhs_client.pong():
login_obj = XiaoHongShuLogin(
@@ -125,8 +125,8 @@ class XiaoHongShuCrawler(AbstractCrawler):
async def search(self) -> None:
"""Search for notes and retrieve their comment information."""
utils.logger.info("[XiaoHongShuCrawler.search] Begin search xiaohongshu keywords")
xhs_limit_count = 20 # xhs limit page fixed value
utils.logger.info("[XiaoHongShuCrawler.search] Begin search Xiaohongshu keywords")
xhs_limit_count = 20 # Xiaohongshu limit page fixed value
if config.CRAWLER_MAX_NOTES_COUNT < xhs_limit_count:
config.CRAWLER_MAX_NOTES_COUNT = xhs_limit_count
start_page = config.START_PAGE
@@ -142,7 +142,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
continue
try:
utils.logger.info(f"[XiaoHongShuCrawler.search] search xhs keyword: {keyword}, page: {page}")
utils.logger.info(f"[XiaoHongShuCrawler.search] search Xiaohongshu keyword: {keyword}, page: {page}")
note_ids: List[str] = []
xsec_tokens: List[str] = []
notes_res = await self.xhs_client.get_note_by_keyword(
@@ -151,9 +151,9 @@ class XiaoHongShuCrawler(AbstractCrawler):
page=page,
sort=(SearchSortType(config.SORT_TYPE) if config.SORT_TYPE != "" else SearchSortType.GENERAL),
)
utils.logger.info(f"[XiaoHongShuCrawler.search] Search notes res:{notes_res}")
utils.logger.info(f"[XiaoHongShuCrawler.search] Search notes response: {notes_res}")
if not notes_res or not notes_res.get("has_more", False):
utils.logger.info("No more content!")
utils.logger.info("[XiaoHongShuCrawler.search] No more content!")
break
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list = [
@@ -184,7 +184,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
async def get_creators_and_notes(self) -> None:
"""Get creator's notes and retrieve their comment information."""
utils.logger.info("[XiaoHongShuCrawler.get_creators_and_notes] Begin get xiaohongshu creators")
utils.logger.info("[XiaoHongShuCrawler.get_creators_and_notes] Begin get Xiaohongshu creators")
for creator_url in config.XHS_CREATOR_ID_LIST:
try:
# Parse creator URL to get user_id and security tokens
@@ -223,9 +223,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
await self.batch_get_note_comments(note_ids, xsec_tokens)
async def fetch_creator_notes_detail(self, note_list: List[Dict]):
"""
Concurrently obtain the specified post list and save the data
"""
"""Concurrently obtain the specified post list and save the data"""
semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
task_list = [
self.get_note_detail_async_task(
@@ -243,11 +241,9 @@ class XiaoHongShuCrawler(AbstractCrawler):
await self.get_notice_media(note_detail)
async def get_specified_notes(self):
"""
Get the information and comments of the specified post
must be specified note_id, xsec_source, xsec_token⚠️⚠️⚠️
Returns:
"""Get the information and comments of the specified post
Note: Must specify note_id, xsec_source, xsec_token
"""
get_note_detail_task_list = []
for full_note_url in config.XHS_SPECIFIED_NOTE_URL_LIST:
@@ -356,8 +352,8 @@ class XiaoHongShuCrawler(AbstractCrawler):
utils.logger.info(f"[XiaoHongShuCrawler.get_comments] Sleeping for {crawl_interval} seconds after fetching comments for note {note_id}")
async def create_xhs_client(self, httpx_proxy: Optional[str]) -> XiaoHongShuClient:
"""Create xhs client"""
utils.logger.info("[XiaoHongShuCrawler.create_xhs_client] Begin create xiaohongshu API client ...")
"""Create Xiaohongshu client"""
utils.logger.info("[XiaoHongShuCrawler.create_xhs_client] Begin create Xiaohongshu API client ...")
cookie_str, cookie_dict = utils.convert_cookies(await self.browser_context.cookies())
xhs_client_obj = XiaoHongShuClient(
proxy=httpx_proxy,
@@ -381,7 +377,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
},
playwright_page=self.context_page,
cookie_dict=cookie_dict,
proxy_ip_pool=self.ip_proxy_pool, # 传递代理池用于自动刷新
proxy_ip_pool=self.ip_proxy_pool, # Pass proxy pool for automatic refresh
)
return xhs_client_obj
@@ -422,9 +418,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
user_agent: Optional[str],
headless: bool = True,
) -> BrowserContext:
"""
使用CDP模式启动浏览器
"""
"""Launch browser using CDP mode"""
try:
self.cdp_manager = CDPBrowserManager()
browser_context = await self.cdp_manager.launch_and_connect(
@@ -434,21 +428,21 @@ class XiaoHongShuCrawler(AbstractCrawler):
headless=headless,
)
# 显示浏览器信息
# Display browser information
browser_info = await self.cdp_manager.get_browser_info()
utils.logger.info(f"[XiaoHongShuCrawler] CDP浏览器信息: {browser_info}")
utils.logger.info(f"[XiaoHongShuCrawler] CDP browser info: {browser_info}")
return browser_context
except Exception as e:
utils.logger.error(f"[XiaoHongShuCrawler] CDP模式启动失败，回退到标准模式: {e}")
# 回退到标准模式
utils.logger.error(f"[XiaoHongShuCrawler] CDP mode launch failed, falling back to standard mode: {e}")
# Fall back to standard mode
chromium = playwright.chromium
return await self.launch_browser(chromium, playwright_proxy, user_agent, headless)
async def close(self):
"""Close browser context"""
# 如果使用CDP模式，需要特殊处理
# Special handling if using CDP mode
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
@@ -464,10 +458,10 @@ class XiaoHongShuCrawler(AbstractCrawler):
await self.get_notice_video(note_detail)
async def get_note_images(self, note_item: Dict):
"""
get note images. please use get_notice_media
:param note_item:
:return:
"""Get note images. Please use get_notice_media
Args:
note_item: Note item dictionary
"""
if not config.ENABLE_GET_MEIDAS:
return
@@ -494,10 +488,10 @@ class XiaoHongShuCrawler(AbstractCrawler):
await xhs_store.update_xhs_note_image(note_id, content, extension_file_name)
async def get_notice_video(self, note_item: Dict):
"""
get note videos. please use get_notice_media
:param note_item:
:return:
"""Get note videos. Please use get_notice_media
Args:
note_item: Note item dictionary
"""
if not config.ENABLE_GET_MEIDAS:
return
media_platform/xhs/extractor.py
+7 -7
View File
@@ -29,16 +29,16 @@ class XiaoHongShuExtractor:
pass
def extract_note_detail_from_html(self, note_id: str, html: str) -> Optional[Dict]:
"""从html中提取笔记详情
"""Extract note details from HTML
Args:
html (str): html字符串
html (str): HTML string
Returns:
Dict: 笔记详情字典
Dict: Note details dictionary
"""
if "noteDetailMap" not in html:
# 这种情况要么是出了验证码了，要么是笔记不存在
# Either a CAPTCHA appeared or the note doesn't exist
return None
state = re.findall(r"window.__INITIAL_STATE__=({.*})</script>", html)[
@@ -50,13 +50,13 @@ class XiaoHongShuExtractor:
return None
def extract_creator_info_from_html(self, html: str) -> Optional[Dict]:
"""从html中提取用户信息
"""Extract user information from HTML
Args:
html (str): html字符串
html (str): HTML string
Returns:
Dict: 用户信息字典
Dict: User information dictionary
"""
match = re.search(
r"<script>window.__INITIAL_STATE__=(.+)<\/script>", html, re.M
media_platform/xhs/field.py
+19 -20
View File
@@ -23,27 +23,27 @@ from typing import NamedTuple
class FeedType(Enum):
# 推荐
# Recommend
RECOMMEND = "homefeed_recommend"
# 穿搭
# Fashion
FASION = "homefeed.fashion_v3"
# 美食
# Food
FOOD = "homefeed.food_v3"
# 彩妆
# Cosmetics
COSMETICS = "homefeed.cosmetics_v3"
# 影视
# Movie and TV
MOVIE = "homefeed.movie_and_tv_v3"
# 职场
# Career
CAREER = "homefeed.career_v3"
# 情感
# Emotion
EMOTION = "homefeed.love_v3"
# 家居
# Home
HOURSE = "homefeed.household_product_v3"
# 游戏
# Gaming
GAME = "homefeed.gaming_v3"
# 旅行
# Travel
TRAVEL = "homefeed.travel_v3"
# 健身
# Fitness
FITNESS = "homefeed.fitness_v3"
@@ -53,28 +53,27 @@ class NoteType(Enum):
class SearchSortType(Enum):
"""search sort type"""
# default
"""Search sort type"""
# Default
GENERAL = "general"
# most popular
# Most popular
MOST_POPULAR = "popularity_descending"
# Latest
LATEST = "time_descending"
class SearchNoteType(Enum):
"""search note type
"""
# default
"""Search note type"""
# Default
ALL = 0
# only video
# Only video
VIDEO = 1
# only image
# Only image
IMAGE = 2
class Note(NamedTuple):
"""note tuple"""
"""Note tuple"""
note_id: str
title: str
desc: str
media_platform/xhs/help.py
+17 -17
View File
@@ -297,13 +297,13 @@ def get_img_urls_by_trace_id(trace_id: str, format_type: str = "png"):
def get_trace_id(img_url: str):
# 浏览器端上传的图片多了 /spectrum/ 这个路径
# Browser-uploaded images have an additional /spectrum/ path
return f"spectrum/{img_url.split('/')[-1]}" if img_url.find("spectrum") != -1 else img_url.split("/")[-1]
def parse_note_info_from_note_url(url: str) -> NoteUrlInfo:
"""
从小红书笔记url中解析出笔记信息
Parse note information from Xiaohongshu note URL
Args:
url: "https://www.xiaohongshu.com/explore/66fad51c000000001b0224b8?xsec_token=AB3rO-QopW5sgrJ41GwN01WCXh6yWPxjSoFI9D5JIMgKw=&xsec_source=pc_search"
Returns:
@@ -318,44 +318,44 @@ def parse_note_info_from_note_url(url: str) -> NoteUrlInfo:
def parse_creator_info_from_url(url: str) -> CreatorUrlInfo:
"""
从小红书创作者主页URL中解析出创作者信息
支持以下格式:
1. 完整URL: "https://www.xiaohongshu.com/user/profile/5eb8e1d400000000010075ae?xsec_token=AB1nWBKCo1vE2HEkfoJUOi5B6BE5n7wVrbdpHoWIj5xHw=&xsec_source=pc_feed"
2. 纯ID: "5eb8e1d400000000010075ae"
Parse creator information from Xiaohongshu creator homepage URL
Supports the following formats:
1. Full URL: "https://www.xiaohongshu.com/user/profile/5eb8e1d400000000010075ae?xsec_token=AB1nWBKCo1vE2HEkfoJUOi5B6BE5n7wVrbdpHoWIj5xHw=&xsec_source=pc_feed"
2. Pure ID: "5eb8e1d400000000010075ae"
Args:
url: 创作者主页URL或user_id
url: Creator homepage URL or user_id
Returns:
CreatorUrlInfo: 包含user_id, xsec_token, xsec_source的对象
CreatorUrlInfo: Object containing user_id, xsec_token, xsec_source
"""
# 如果是纯ID格式(24位十六进制字符),直接返回
# If it's a pure ID format (24 hexadecimal characters), return directly
if len(url) == 24 and all(c in "0123456789abcdef" for c in url):
return CreatorUrlInfo(user_id=url, xsec_token="", xsec_source="")
# 从URL中提取user_id: /user/profile/xxx
# Extract user_id from URL: /user/profile/xxx
import re
user_pattern = r'/user/profile/([^/?]+)'
match = re.search(user_pattern, url)
if match:
user_id = match.group(1)
# 提取xsec_token和xsec_source参数
# Extract xsec_token and xsec_source parameters
params = extract_url_params_to_dict(url)
xsec_token = params.get("xsec_token", "")
xsec_source = params.get("xsec_source", "")
return CreatorUrlInfo(user_id=user_id, xsec_token=xsec_token, xsec_source=xsec_source)
raise ValueError(f"无法从URL中解析出创作者信息: {url}")
raise ValueError(f"Unable to parse creator info from URL: {url}")
if __name__ == '__main__':
_img_url = "https://sns-img-bd.xhscdn.com/7a3abfaf-90c1-a828-5de7-022c80b92aa3"
# 获取一个图片地址在多个cdn下的url地址
# Get image URL addresses under multiple CDNs for a single image
# final_img_urls = get_img_urls_by_trace_id(get_trace_id(_img_url))
final_img_url = get_img_url_by_trace_id(get_trace_id(_img_url))
print(final_img_url)
# 测试创作者URL解析
print("\n=== 创作者URL解析测试 ===")
# Test creator URL parsing
print("\n=== Creator URL Parsing Test ===")
test_creator_urls = [
"https://www.xiaohongshu.com/user/profile/5eb8e1d400000000010075ae?xsec_token=AB1nWBKCo1vE2HEkfoJUOi5B6BE5n7wVrbdpHoWIj5xHw=&xsec_source=pc_feed",
"5eb8e1d400000000010075ae",
@@ -364,7 +364,7 @@ if __name__ == '__main__':
try:
result = parse_creator_info_from_url(url)
print(f"✓ URL: {url[:80]}...")
print(f" 结果: {result}\n")
print(f" Result: {result}\n")
except Exception as e:
print(f"✗ URL: {url}")
print(f" 错误: {e}\n")
print(f" Error: {e}\n")
media_platform/xhs/login.py
+11 -11
View File
@@ -57,7 +57,7 @@ class XiaoHongShuLogin(AbstractLogin):
"""
if "请通过验证" in await self.context_page.content():
utils.logger.info("[XiaoHongShuLogin.check_login_state] 登录过程中出现验证码，请手动验证")
utils.logger.info("[XiaoHongShuLogin.check_login_state] CAPTCHA appeared during login, please verify manually")
current_cookie = await self.browser_context.cookies()
_, cookie_dict = utils.convert_cookies(current_cookie)
@@ -83,14 +83,14 @@ class XiaoHongShuLogin(AbstractLogin):
utils.logger.info("[XiaoHongShuLogin.login_by_mobile] Begin login xiaohongshu by mobile ...")
await asyncio.sleep(1)
try:
# 小红书进入首页后，有可能不会自动弹出登录框，需要手动点击登录按钮
# After entering Xiaohongshu homepage, the login dialog may not pop up automatically, need to manually click login button
login_button_ele = await self.context_page.wait_for_selector(
selector="xpath=//*[@id='app']/div[1]/div[2]/div[1]/ul/div[1]/button",
timeout=5000
)
await login_button_ele.click()
# 弹窗的登录对话框也有两种形态，一种是直接可以看到手机号和验证码的
# 另一种是需要点击切换到手机登录的
# The login dialog has two forms: one shows phone number and verification code directly
# The other requires clicking to switch to phone login
element = await self.context_page.wait_for_selector(
selector='xpath=//div[@class="login-container"]//div[@class="other-method"]/div[1]',
timeout=5000
@@ -106,11 +106,11 @@ class XiaoHongShuLogin(AbstractLogin):
await asyncio.sleep(0.5)
send_btn_ele = await login_container_ele.query_selector("label.auth-code > span")
await send_btn_ele.click() # 点击发送验证码
await send_btn_ele.click() # Click to send verification code
sms_code_input_ele = await login_container_ele.query_selector("label.auth-code > input")
submit_btn_ele = await login_container_ele.query_selector("div.input-container > button")
cache_client = CacheFactory.create_cache(config.CACHE_TYPE_MEMORY)
max_get_sms_code_time = 60 * 2 # 最长获取验证码的时间为2分钟
max_get_sms_code_time = 60 * 2 # Maximum time to get verification code is 2 minutes
no_logged_in_session = ""
while max_get_sms_code_time > 0:
utils.logger.info(f"[XiaoHongShuLogin.login_by_mobile] get sms code from redis remaining time {max_get_sms_code_time}s ...")
@@ -125,15 +125,15 @@ class XiaoHongShuLogin(AbstractLogin):
_, cookie_dict = utils.convert_cookies(current_cookie)
no_logged_in_session = cookie_dict.get("web_session")
await sms_code_input_ele.fill(value=sms_code_value.decode()) # 输入短信验证码
await sms_code_input_ele.fill(value=sms_code_value.decode()) # Enter SMS verification code
await asyncio.sleep(0.5)
agree_privacy_ele = self.context_page.locator("xpath=//div[@class='agreements']//*[local-name()='svg']")
await agree_privacy_ele.click() # 点击同意隐私协议
await agree_privacy_ele.click() # Click to agree to privacy policy
await asyncio.sleep(0.5)
await submit_btn_ele.click() # 点击登录
await submit_btn_ele.click() # Click login
# todo ... 应该还需要检查验证码的正确性有可能输入的验证码不正确
# TODO: Should also check if the verification code is correct, as it may be incorrect
break
try:
@@ -196,7 +196,7 @@ class XiaoHongShuLogin(AbstractLogin):
"""login xiaohongshu website by cookies"""
utils.logger.info("[XiaoHongShuLogin.login_by_cookies] Begin login xiaohongshu by cookie ...")
for key, value in utils.convert_str_cookie_to_dict(self.cookie_str).items():
if key != "web_session": # only set web_session cookie attr
if key != "web_session": # Only set web_session cookie attribute
continue
await self.browser_context.add_cookies([{
'name': key,
media_platform/xhs/playwright_sign.py
+44 -44
View File
@@ -16,7 +16,7 @@
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
# 通过 Playwright 注入调用 window.mnsv2 生成小红书签名
# Generate Xiaohongshu signature by calling window.mnsv2 via Playwright injection
import hashlib
import json
@@ -30,18 +30,18 @@ from .xhs_sign import b64_encode, encode_utf8, get_trace_id, mrc
def _build_sign_string(uri: str, data: Optional[Union[Dict, str]] = None, method: str = "POST") -> str:
"""构建待签名字符串
"""Build string to be signed
Args:
uri: API路径
data: 请求数据
method: 请求方法 (GET 或 POST)
uri: API path
data: Request data
method: Request method (GET or POST)
Returns:
待签名字符串
String to be signed
"""
if method.upper() == "POST":
# POST 请求使用 JSON 格式
# POST request uses JSON format
c = uri
if data is not None:
if isinstance(data, dict):
@@ -50,10 +50,10 @@ def _build_sign_string(uri: str, data: Optional[Union[Dict, str]] = None, method
c += data
return c
else:
# GET 请求使用查询字符串格式
# GET request uses query string format
if not data or (isinstance(data, dict) and len(data) == 0):
return uri
if isinstance(data, dict):
params = []
for key in data.keys():
@@ -64,8 +64,8 @@ def _build_sign_string(uri: str, data: Optional[Union[Dict, str]] = None, method
value_str = str(value)
else:
value_str = ""
# 使用URL编码（safe参数保留某些字符不编码）
# 注意：httpx会对逗号、等号等字符进行编码，我们也需要同样处理
# Use URL encoding (safe parameter preserves certain characters from encoding)
# Note: httpx will encode commas, equals signs, etc., we need to handle the same way
value_str = quote(value_str, safe='')
params.append(f"{key}={value_str}")
return f"{uri}?{'&'.join(params)}"
@@ -75,12 +75,12 @@ def _build_sign_string(uri: str, data: Optional[Union[Dict, str]] = None, method
def _md5_hex(s: str) -> str:
"""计算 MD5 哈希值"""
"""Calculate MD5 hash value"""
return hashlib.md5(s.encode("utf-8")).hexdigest()
def _build_xs_payload(x3_value: str, data_type: str = "object") -> str:
"""构建 x-s 签名"""
"""Build x-s signature"""
s = {
"x0": "4.2.1",
"x1": "xhs-pc-web",
@@ -92,7 +92,7 @@ def _build_xs_payload(x3_value: str, data_type: str = "object") -> str:
def _build_xs_common(a1: str, b1: str, x_s: str, x_t: str) -> str:
"""构建 x-s-common 请求头"""
"""Build x-s-common request header"""
payload = {
"s0": 3,
"s1": "",
@@ -113,7 +113,7 @@ def _build_xs_common(a1: str, b1: str, x_s: str, x_t: str) -> str:
async def get_b1_from_localstorage(page: Page) -> str:
"""从 localStorage 获取 b1 值"""
"""Get b1 value from localStorage"""
try:
local_storage = await page.evaluate("() => window.localStorage")
return local_storage.get("b1", "")
@@ -123,15 +123,15 @@ async def get_b1_from_localstorage(page: Page) -> str:
async def call_mnsv2(page: Page, sign_str: str, md5_str: str) -> str:
"""
通过 playwright 调用 window.mnsv2 函数
Call window.mnsv2 function via playwright
Args:
page: playwright Page 对象
sign_str: 待签名字符串 (uri + JSON.stringify(data))
md5_str: sign_str 的 MD5 哈希值
page: playwright Page object
sign_str: String to be signed (uri + JSON.stringify(data))
md5_str: MD5 hash value of sign_str
Returns:
mnsv2 返回的签名字符串
Signature string returned by mnsv2
"""
sign_str_escaped = sign_str.replace("\\", "\\\\").replace("'", "\\'").replace("\n", "\\n")
md5_str_escaped = md5_str.replace("\\", "\\\\").replace("'", "\\'")
@@ -150,16 +150,16 @@ async def sign_xs_with_playwright(
method: str = "POST",
) -> str:
"""
通过 playwright 注入生成 x-s 签名
Generate x-s signature via playwright injection
Args:
page: playwright Page 对象（必须已打开小红书页面）
uri: API 路径，如 "/api/sns/web/v1/search/notes"
data: 请求数据（GET 的 params 或 POST 的 payload）
method: 请求方法 (GET 或 POST)
page: playwright Page object (must have Xiaohongshu page open)
uri: API path, e.g., "/api/sns/web/v1/search/notes"
data: Request data (GET params or POST payload)
method: Request method (GET or POST)
Returns:
x-s 签名字符串
x-s signature string
"""
sign_str = _build_sign_string(uri, data, method)
md5_str = _md5_hex(sign_str)
@@ -176,17 +176,17 @@ async def sign_with_playwright(
method: str = "POST",
) -> Dict[str, Any]:
"""
通过 playwright 生成完整的签名请求头
Generate complete signature request headers via playwright
Args:
page: playwright Page 对象（必须已打开小红书页面）
uri: API 路径
data: 请求数据
a1: cookie 中的 a1 值
method: 请求方法 (GET 或 POST)
page: playwright Page object (must have Xiaohongshu page open)
uri: API path
data: Request data
a1: a1 value from cookie
method: Request method (GET or POST)
Returns:
包含 x-s, x-t, x-s-common, x-b3-traceid 的字典
Dictionary containing x-s, x-t, x-s-common, x-b3-traceid
"""
b1 = await get_b1_from_localstorage(page)
x_s = await sign_xs_with_playwright(page, uri, data, method)
@@ -208,23 +208,23 @@ async def pre_headers_with_playwright(
payload: Optional[Dict] = None,
) -> Dict[str, str]:
"""
使用 playwright 注入方式生成请求头签名
可直接替换 client.py 中的 _pre_headers 方法
Generate request header signature using playwright injection method
Can directly replace _pre_headers method in client.py
Args:
page: playwright Page 对象
url: 请求 URL
cookie_dict: cookie 字典
params: GET 请求参数
payload: POST 请求参数
page: playwright Page object
url: Request URL
cookie_dict: Cookie dictionary
params: GET request parameters
payload: POST request parameters
Returns:
签名后的请求头字典
Signed request header dictionary
"""
a1_value = cookie_dict.get("a1", "")
uri = urlparse(url).path
# 确定请求数据和方法
# Determine request data and method
if params is not None:
data = params
method = "GET"
media_platform/xhs/xhs_sign.py
+13 -13
View File
@@ -16,19 +16,19 @@
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
# 小红书签名算法核心函数
# 用于 playwright 注入方式生成签名
# Xiaohongshu signature algorithm core functions
# Used for generating signatures via playwright injection
import ctypes
import random
from urllib.parse import quote
# 自定义 Base64 字符表
# 标准 Base64: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/
# 小红书打乱顺序用于混淆
# Custom Base64 character table
# Standard Base64: ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/
# Xiaohongshu shuffled order for obfuscation
BASE64_CHARS = list("ZmserbBoHQtNP+wOcza/LpngG8yJq42KWYj0DSfdikx3VT16IlUAFM97hECvuRX5")
# CRC32 查表
# CRC32 lookup table
CRC32_TABLE = [
0, 1996959894, 3993919788, 2567524794, 124634137, 1886057615, 3915621685,
2657392035, 249268274, 2044508324, 3772115230, 2547177864, 162941995,
@@ -77,14 +77,14 @@ CRC32_TABLE = [
def _right_shift_unsigned(num: int, bit: int = 0) -> int:
"""JavaScript 无符号右移 (>>>) 的 Python 实现"""
"""Python implementation of JavaScript unsigned right shift (>>>)"""
val = ctypes.c_uint32(num).value >> bit
MAX32INT = 4294967295
return (val + (MAX32INT + 1)) % (2 * (MAX32INT + 1)) - MAX32INT - 1
def mrc(e: str) -> int:
"""CRC32 变体，用于 x-s-common 的 x9 字段"""
"""CRC32 variant, used for x9 field in x-s-common"""
o = -1
for n in range(min(57, len(e))):
o = CRC32_TABLE[(o & 255) ^ ord(e[n])] ^ _right_shift_unsigned(o, 8)
@@ -92,7 +92,7 @@ def mrc(e: str) -> int:
def _triplet_to_base64(e: int) -> str:
"""将 24 位整数转换为 4 个 Base64 字符"""
"""Convert 24-bit integer to 4 Base64 characters"""
return (
BASE64_CHARS[(e >> 18) & 63]
+ BASE64_CHARS[(e >> 12) & 63]
@@ -102,7 +102,7 @@ def _triplet_to_base64(e: int) -> str:
def _encode_chunk(data: list, start: int, end: int) -> str:
"""编码数据块"""
"""Encode data chunk"""
result = []
for i in range(start, end, 3):
c = ((data[i] << 16) & 0xFF0000) + ((data[i + 1] << 8) & 0xFF00) + (data[i + 2] & 0xFF)
@@ -111,7 +111,7 @@ def _encode_chunk(data: list, start: int, end: int) -> str:
def encode_utf8(s: str) -> list:
"""将字符串编码为 UTF-8 字节列表"""
"""Encode string to UTF-8 byte list"""
encoded = quote(s, safe="~()*!.'")
result = []
i = 0
@@ -126,7 +126,7 @@ def encode_utf8(s: str) -> list:
def b64_encode(data: list) -> str:
"""自定义 Base64 编码"""
"""Custom Base64 encoding"""
length = len(data)
remainder = length % 3
chunks = []
@@ -148,5 +148,5 @@ def b64_encode(data: list) -> str:
def get_trace_id() -> str:
"""生成链路追踪 trace id"""
"""Generate trace id for link tracing"""
return "".join(random.choice("abcdef0123456789") for _ in range(16))
media_platform/zhihu/client.py
+49 -49
View File
@@ -60,14 +60,14 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
self.default_headers = headers
self.cookie_dict = cookie_dict
self._extractor = ZhihuExtractor()
# 初始化代理池（来自 ProxyRefreshMixin）
# Initialize proxy pool (from ProxyRefreshMixin)
self.init_proxy_pool(proxy_ip_pool)
async def _pre_headers(self, url: str) -> Dict:
"""
请求头参数签名
Sign request headers
Args:
url: 请求的URL需要包含请求的参数
url: Request URL with query parameters
Returns:
"""
@@ -83,16 +83,16 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
@retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
async def request(self, method, url, **kwargs) -> Union[str, Any]:
"""
封装httpx的公共请求方法，对请求响应做一些处理
Wrapper for httpx common request method with response handling
Args:
method: 请求方法
url: 请求的URL
**kwargs: 其他请求参数，例如请求头、请求体等
method: Request method
url: Request URL
**kwargs: Other request parameters such as headers, body, etc.
Returns:
"""
# 每次请求前检测代理是否过期
# Check if proxy is expired before each request
await self._refresh_proxy_if_expired()
# return response.text
@@ -105,7 +105,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
utils.logger.error(f"[ZhiHuClient.request] Requset Url: {url}, Request error: {response.text}")
if response.status_code == 403:
raise ForbiddenError(response.text)
elif response.status_code == 404: # 如果一个content没有评论也是404
elif response.status_code == 404: # Content without comments also returns 404
return {}
raise DataFetchError(response.text)
@@ -124,10 +124,10 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def get(self, uri: str, params=None, **kwargs) -> Union[Response, Dict, str]:
"""
GET请求，对请求头签名
GET request with header signing
Args:
uri: 请求路由
params: 请求参数
uri: Request URI
params: Request parameters
Returns:
@@ -141,7 +141,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def pong(self) -> bool:
"""
用于检查登录态是否失效了
Check if login status is still valid
Returns:
"""
@@ -161,9 +161,9 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def update_cookies(self, browser_context: BrowserContext):
"""
API客户端提供的更新cookies方法，一般情况下登录成功后会调用此方法
Update cookies method provided by API client, typically called after successful login
Args:
browser_context: 浏览器上下文对象
browser_context: Browser context object
Returns:
@@ -174,7 +174,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def get_current_user_info(self) -> Dict:
"""
获取当前登录用户信息
Get current logged-in user information
Returns:
"""
@@ -191,14 +191,14 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
search_time: SearchTime = SearchTime.DEFAULT,
) -> List[ZhihuContent]:
"""
根据关键词搜索
Search by keyword
Args:
keyword: 关键词
page: 第几页
page_size: 分页size
sort: 排序
note_type: 搜索结果类型
search_time: 搜索多久时间的结果
keyword: Search keyword
page: Page number
page_size: Page size
sort: Sorting method
note_type: Search result type
search_time: Time range for search results
Returns:
@@ -232,10 +232,10 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
order_by: str = "score",
) -> Dict:
"""
获取内容的一级评论
Get root-level comments for content
Args:
content_id: 内容ID
content_type: 内容类型(answer, article, zvideo)
content_id: Content ID
content_type: Content type (answer, article, zvideo)
offset:
limit:
order_by:
@@ -262,7 +262,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
order_by: str = "sort",
) -> Dict:
"""
获取一级评论下的子评论
Get child comments under a root comment
Args:
root_comment_id:
offset:
@@ -287,11 +287,11 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
callback: Optional[Callable] = None,
) -> List[ZhihuComment]:
"""
获取指定帖子下的所有一级评论，该方法会一直查找一个帖子下的所有评论信息
Get all root-level comments for a specified post, this method will retrieve all comment information under a post
Args:
content: 内容详情对象(问题｜文章｜视频)
crawl_interval: 爬取一次笔记的延迟单位（秒）
callback: 一次笔记爬取结束后
content: Content detail object (question|article|video)
crawl_interval: Crawl delay interval in seconds
callback: Callback after completing one crawl
Returns:
@@ -328,12 +328,12 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
callback: Optional[Callable] = None,
) -> List[ZhihuComment]:
"""
获取指定评论下的所有子评论
Get all sub-comments under specified comments
Args:
content: 内容详情对象(问题｜文章｜视频)
comments: 评论列表
crawl_interval: 爬取一次笔记的延迟单位（秒）
callback: 一次笔记爬取结束后
content: Content detail object (question|article|video)
comments: Comment list
crawl_interval: Crawl delay interval in seconds
callback: Callback after completing one crawl
Returns:
@@ -370,7 +370,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def get_creator_info(self, url_token: str) -> Optional[ZhihuCreator]:
"""
获取创作者信息
Get creator information
Args:
url_token:
@@ -383,7 +383,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def get_creator_answers(self, url_token: str, offset: int = 0, limit: int = 20) -> Dict:
"""
获取创作者的回答
Get creator's answers
Args:
url_token:
offset:
@@ -405,7 +405,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def get_creator_articles(self, url_token: str, offset: int = 0, limit: int = 20) -> Dict:
"""
获取创作者的文章
Get creator's articles
Args:
url_token:
offset:
@@ -426,7 +426,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def get_creator_videos(self, url_token: str, offset: int = 0, limit: int = 20) -> Dict:
"""
获取创作者的视频
Get creator's videos
Args:
url_token:
offset:
@@ -446,11 +446,11 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def get_all_anwser_by_creator(self, creator: ZhihuCreator, crawl_interval: float = 1.0, callback: Optional[Callable] = None) -> List[ZhihuContent]:
"""
获取创作者的所有回答
Get all answers by creator
Args:
creator: 创作者信息
crawl_interval: 爬取一次笔记的延迟单位（秒）
callback: 一次笔记爬取结束后
creator: Creator information
crawl_interval: Crawl delay interval in seconds
callback: Callback after completing one crawl
Returns:
@@ -481,7 +481,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
callback: Optional[Callable] = None,
) -> List[ZhihuContent]:
"""
获取创作者的所有文章
Get all articles by creator
Args:
creator:
crawl_interval:
@@ -515,7 +515,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
callback: Optional[Callable] = None,
) -> List[ZhihuContent]:
"""
获取创作者的所有视频
Get all videos by creator
Args:
creator:
crawl_interval:
@@ -548,7 +548,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
answer_id: str,
) -> Optional[ZhihuContent]:
"""
获取回答信息
Get answer information
Args:
question_id:
answer_id:
@@ -562,7 +562,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def get_article_info(self, article_id: str) -> Optional[ZhihuContent]:
"""
获取文章信息
Get article information
Args:
article_id:
@@ -575,7 +575,7 @@ class ZhiHuClient(AbstractApiClient, ProxyRefreshMixin):
async def get_video_info(self, video_id: str) -> Optional[ZhihuContent]:
"""
获取视频信息
Get video information
Args:
video_id:
media_platform/zhihu/core.py
+16 -16
View File
@@ -61,7 +61,7 @@ class ZhihuCrawler(AbstractCrawler):
self.user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
self._extractor = ZhihuExtractor()
self.cdp_manager = None
self.ip_proxy_pool = None # 代理IP池，用于代理自动刷新
self.ip_proxy_pool = None # Proxy IP pool for automatic proxy refresh
async def start(self) -> None:
"""
@@ -80,9 +80,9 @@ class ZhihuCrawler(AbstractCrawler):
)
async with async_playwright() as playwright:
# 根据配置选择启动模式
# Choose launch mode based on configuration
if config.ENABLE_CDP_MODE:
utils.logger.info("[ZhihuCrawler] 使用CDP模式启动浏览器")
utils.logger.info("[ZhihuCrawler] Launching browser in CDP mode")
self.browser_context = await self.launch_browser_with_cdp(
playwright,
playwright_proxy_format,
@@ -90,7 +90,7 @@ class ZhihuCrawler(AbstractCrawler):
headless=config.CDP_HEADLESS,
)
else:
utils.logger.info("[ZhihuCrawler] 使用标准模式启动浏览器")
utils.logger.info("[ZhihuCrawler] Launching browser in standard mode")
# Launch a browser context.
chromium = playwright.chromium
self.browser_context = await self.launch_browser(
@@ -117,9 +117,9 @@ class ZhihuCrawler(AbstractCrawler):
browser_context=self.browser_context
)
# 知乎的搜索接口需要打开搜索页面之后cookies才能访问API，单独的首页不行
# Zhihu's search API requires opening the search page first to access cookies, homepage alone won't work
utils.logger.info(
"[ZhihuCrawler.start] Zhihu跳转到搜索页面获取搜索页面的Cookies，该过程需要5秒左右"
"[ZhihuCrawler.start] Zhihu navigating to search page to get search page cookies, this process takes about 5 seconds"
)
await self.context_page.goto(
f"{self.index_url}/search?q=python&search_source=Guess&utm_content=search_hot&type=content"
@@ -273,7 +273,7 @@ class ZhihuCrawler(AbstractCrawler):
)
await zhihu_store.save_creator(creator=createor_info)
# 默认只提取回答信息，如果需要文章和视频，把下面的注释打开即可
# By default, only answer information is extracted, uncomment below if articles and videos are needed
# Get all anwser information of the creator
all_content_list = await self.zhihu_client.get_all_anwser_by_creator(
@@ -315,7 +315,7 @@ class ZhihuCrawler(AbstractCrawler):
utils.logger.info(
f"[ZhihuCrawler.get_specified_notes] Begin get specified note {full_note_url}"
)
# judge note type
# Judge note type
note_type: str = judge_zhihu_url(full_note_url)
if note_type == constant.ANSWER_NAME:
question_id = full_note_url.split("/")[-3]
@@ -412,7 +412,7 @@ class ZhihuCrawler(AbstractCrawler):
},
playwright_page=self.context_page,
cookie_dict=cookie_dict,
proxy_ip_pool=self.ip_proxy_pool, # 传递代理池用于自动刷新
proxy_ip_pool=self.ip_proxy_pool, # Pass proxy pool for automatic refresh
)
return zhihu_client_obj
@@ -440,7 +440,7 @@ class ZhihuCrawler(AbstractCrawler):
proxy=playwright_proxy, # type: ignore
viewport={"width": 1920, "height": 1080},
user_agent=user_agent,
channel="chrome", # 使用系统的Chrome稳定版
channel="chrome", # Use system Chrome stable version
)
return browser_context
else:
@@ -458,7 +458,7 @@ class ZhihuCrawler(AbstractCrawler):
headless: bool = True,
) -> BrowserContext:
"""
使用CDP模式启动浏览器
Launch browser using CDP mode
"""
try:
self.cdp_manager = CDPBrowserManager()
@@ -469,15 +469,15 @@ class ZhihuCrawler(AbstractCrawler):
headless=headless,
)
# 显示浏览器信息
# Display browser information
browser_info = await self.cdp_manager.get_browser_info()
utils.logger.info(f"[ZhihuCrawler] CDP浏览器信息: {browser_info}")
utils.logger.info(f"[ZhihuCrawler] CDP browser info: {browser_info}")
return browser_context
except Exception as e:
utils.logger.error(f"[ZhihuCrawler] CDP模式启动失败，回退到标准模式: {e}")
# 回退到标准模式
utils.logger.error(f"[ZhihuCrawler] CDP mode launch failed, falling back to standard mode: {e}")
# Fall back to standard mode
chromium = playwright.chromium
return await self.launch_browser(
chromium, playwright_proxy, user_agent, headless
@@ -485,7 +485,7 @@ class ZhihuCrawler(AbstractCrawler):
async def close(self):
"""Close browser context"""
# 如果使用CDP模式，需要特殊处理
# Special handling if using CDP mode
if self.cdp_manager:
await self.cdp_manager.cleanup()
self.cdp_manager = None
media_platform/zhihu/field.py
+17 -17
View File
@@ -26,31 +26,31 @@ from constant import zhihu as zhihu_constant
class SearchTime(Enum):
"""
搜索时间范围
Search time range
"""
DEFAULT = "" # 不限时间
ONE_DAY = "a_day" # 一天内
ONE_WEEK = "a_week" # 一周内
ONE_MONTH = "a_month" # 一个月内
THREE_MONTH = "three_months" # 三个月内
HALF_YEAR = "half_a_year" # 半年内
ONE_YEAR = "a_year" # 一年内
DEFAULT = "" # No time limit
ONE_DAY = "a_day" # Within one day
ONE_WEEK = "a_week" # Within one week
ONE_MONTH = "a_month" # Within one month
THREE_MONTH = "three_months" # Within three months
HALF_YEAR = "half_a_year" # Within half a year
ONE_YEAR = "a_year" # Within one year
class SearchType(Enum):
"""
搜索结果类型
Search result type
"""
DEFAULT = "" # 不限类型
ANSWER = zhihu_constant.ANSWER_NAME # 只看回答
ARTICLE = zhihu_constant.ARTICLE_NAME # 只看文章
VIDEO = zhihu_constant.VIDEO_NAME # 只看视频
DEFAULT = "" # No type limit
ANSWER = zhihu_constant.ANSWER_NAME # Answers only
ARTICLE = zhihu_constant.ARTICLE_NAME # Articles only
VIDEO = zhihu_constant.VIDEO_NAME # Videos only
class SearchSort(Enum):
"""
搜索结果排序
Search result sorting
"""
DEFAULT = "" # 综合排序
UPVOTED_COUNT = "upvoted_count" # 最多赞同
CREATE_TIME = "created_time" # 最新发布
DEFAULT = "" # Default sorting
UPVOTED_COUNT = "upvoted_count" # Most upvoted
CREATE_TIME = "created_time" # Latest published
media_platform/zhihu/help.py
+4 -4
View File
@@ -168,7 +168,7 @@ class ZhihuExtractor:
"""
res = ZhihuContent()
if "video" in zvideo and isinstance(zvideo.get("video"), dict): # 说明是从创作者主页的视频列表接口来的
if "video" in zvideo and isinstance(zvideo.get("video"), dict): # This indicates data from the creator's homepage video list API
res.content_url = f"{zhihu_constant.ZHIHU_URL}/zvideo/{res.content_id}"
res.created_time = zvideo.get("published_at")
res.updated_time = zvideo.get("updated_at")
@@ -318,11 +318,11 @@ class ZhihuExtractor:
"""
if gender == 1:
return "男"
return "Male"
elif gender == 0:
return "女"
return "Female"
else:
return "未知"
return "Unknown"
def extract_creator(self, user_url_token: str, html_content: str) -> Optional[ZhihuCreator]: