From cffb407d15a3b10af7b3e177666547171f63b4cd Mon Sep 17 00:00:00 2001 From: butubb <1422726308@qq.com> Date: Wed, 7 Oct 2026 11:08:11 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E7=94=9F=E4=BA=A7=E5=BA=93=E5=85=8B?= =?UTF-8?q?=E9=9A=86=E8=84=9A=E6=9C=AC=20+=20=E5=AE=B9=E5=99=A8=E8=A1=A5?= =?UTF-8?q?=20Node=20=E8=BF=90=E8=A1=8C=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - tools/clone_database.py: 同服务器跨库克隆并建立独立应用账号的 provisioning 脚本。本机与服务器都没有 mysql 客户端，所以走 INSERT...SELECT 而非 dump/reload。刻意只读写命令行指定的两个 schema，且给应用单开账号而不是复用管理员 - Dockerfile: 补 nodejs。douyin/help.py 在模块导入阶段就执行 execjs.compile(libs/douyin.js)，缺 JS 运行时会抛 RuntimeUnavailableError；而 main.py 要导入全部 7 个平台，于是整个应用连带环境自检一起挂掉 已在 192.168.20.220 上验证：7 个平台导入全部通过。 --- Dockerfile | 7 +- tools/clone_database.py | 200 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 tools/clone_database.py diff --git a/Dockerfile b/Dockerfile index e0196e0..308d272 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,10 @@ ENV PYTHONUNBUFFERED=1 \ # at import time, and tools/utils.py reaches cv2 through slider_util, so without # them the *application* fails to import, not just some image utility. tzdata is # here because TZ=Asia/Shanghai is silently ignored without it, which would put -# every stored timestamp in UTC. +# every stored timestamp in UTC. nodejs is for PyExecJS: douyin/help.py compiles +# libs/douyin.js at *import* time, and because main.py imports every platform, +# that single platform being importable-or-not decides whether the whole app +# (and the environment self-check) comes up. # # The pip mirror is set for the same reason as the apt one: this host's route to # the public index is slow. @@ -41,7 +44,7 @@ RUN set -eux; \ apt-get install -y --no-install-recommends \ build-essential pkg-config default-libmysqlclient-dev \ libgl1 libglib2.0-0 libsm6 libxext6 libxrender1 libxcb1 libgomp1 \ - tzdata; \ + tzdata nodejs; \ rm -rf /var/lib/apt/lists/* WORKDIR /app diff --git a/tools/clone_database.py b/tools/clone_database.py new file mode 100644 index 0000000..c11489e --- /dev/null +++ b/tools/clone_database.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Clone one schema to a new database on the same MySQL server, and provision a +scoped account for it. + +Written for the production cut-over: the monitor data lives in `mediacrawler` and +the server deployment reads `mediacrawler_prod`. Both sit on the same host, so the +copy is a cross-schema ``INSERT ... SELECT`` rather than a dump-and-reload -- and +since neither this workstation nor the server has a mysql client, that is also +the only option available. + +Two things this deliberately does NOT do, both because the server hosts ~22 +unrelated production databases and the provisioning account is a full admin: + +* it never writes outside the two schemas named on the command line; +* it hands the application its own account, scoped to the new schema, rather than + reusing that admin account for the app. + +Foreign keys are disabled only for the duration of the copy. Source and target +definitions are identical, so ordering is the sole thing at stake, and turning +the checks off is what makes an arbitrary table order safe. +""" + +import argparse +import asyncio +import secrets +import string +import sys +from pathlib import Path + +import aiomysql + +PASSWORD_ALPHABET = string.ascii_letters + string.digits + + +def generate_password(length: int = 32) -> str: + """Alphanumeric only: it ends up in a .env value and a SQL literal, and a + symbol that needs escaping in either place is a support ticket waiting.""" + return "".join(secrets.choice(PASSWORD_ALPHABET) for _ in range(length)) + + +async def _admin_conn(args, db=None): + return await aiomysql.connect( + host=args.host, + port=args.port, + user=args.admin_user, + password=args.admin_password, + db=db, + charset="utf8mb4", + autocommit=True, + ) + + +async def create_schema(args) -> None: + conn = await _admin_conn(args) + try: + cur = await conn.cursor() + await cur.execute( + f"CREATE DATABASE IF NOT EXISTS `{args.target_db}` " + "DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" + ) + await cur.execute( + "SELECT DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME " + "FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = %s", + (args.target_db,), + ) + charset, collation = await cur.fetchone() + print(f"[库] {args.target_db} {charset} / {collation}") + await cur.close() + finally: + conn.close() + + +async def provision_app_user(args, password: str) -> None: + conn = await _admin_conn(args) + try: + cur = await conn.cursor() + # The host is bound as a parameter instead of written as a literal '%': + # aiomysql applies %-substitution whenever arguments are supplied, so a + # bare '%' in the SQL is read as a format specifier and raises + # "unsupported format character". + await cur.execute( + "CREATE USER IF NOT EXISTS %s@%s IDENTIFIED BY %s", + (args.app_user, "%", password), + ) + # Idempotent re-runs: if the account already existed, make the password + # the one we are about to write into .env rather than a stale one. + await cur.execute( + "ALTER USER %s@%s IDENTIFIED BY %s", (args.app_user, "%", password) + ) + await cur.execute( + f"GRANT ALL PRIVILEGES ON `{args.target_db}`.* TO %s@%s", + (args.app_user, "%"), + ) + await cur.execute("FLUSH PRIVILEGES") + print(f"[账号] {args.app_user}@% 授权范围仅 `{args.target_db}`.*") + await cur.close() + finally: + conn.close() + + +async def copy_tables(args) -> None: + src = await _admin_conn(args, db=args.source_db) + dst = await _admin_conn(args, db=args.target_db) + try: + scur = await src.cursor() + await scur.execute("SHOW TABLES") + tables = [row[0] for row in await scur.fetchall()] + print(f"[表] 源库共 {len(tables)} 张") + + dcur = await dst.cursor() + # Order is arbitrary on purpose -- checks are off for the whole copy, so + # a table may safely be created before the table it references. + await dcur.execute("SET FOREIGN_KEY_CHECKS=0") + + for table in tables: + await scur.execute(f"SHOW CREATE TABLE `{table}`") + ddl = (await scur.fetchone())[1] + # `IF NOT EXISTS` makes the script re-runnable after a partial run. + await dcur.execute(ddl.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1)) + + await dcur.execute(f"DELETE FROM `{table}`") + await dcur.execute( + f"INSERT INTO `{table}` SELECT * FROM `{args.source_db}`.`{table}`" + ) + copied = dcur.rowcount + + await scur.execute(f"SELECT COUNT(*) FROM `{table}`") + expected = (await scur.fetchone())[0] + flag = "OK" if copied == expected else "!! 行数不符" + print(f" {table:<26} {copied:>6} / {expected:<6} {flag}") + + await dcur.execute("SET FOREIGN_KEY_CHECKS=1") + await dcur.close() + await scur.close() + finally: + src.close() + dst.close() + + +def write_env(path: Path, args, password: str) -> None: + path.write_text( + f"""# 服务器部署环境配置（已被 .gitignore 忽略，不会提交） +# +# 由 tools/clone_database.py 生成。库名改了之后必须同时确认账号对该库有权限： +# 应用启动时会校验「实际连到的库」是否等于下面的 MYSQL_DB_NAME，不符会拒绝启动。 +MC_HOST=0.0.0.0 +MC_PORT=18051 + +# --- 数据库（监控层） --- +MYSQL_DB_HOST={args.host} +MYSQL_DB_PORT={args.port} +MYSQL_DB_USER={args.app_user} +MYSQL_DB_PWD={password} +MYSQL_DB_NAME={args.target_db} + +# 登录鉴权：留空则首次启动自动生成随机密码并打印在启动日志里 +# MC_PASSWORD= + +# 面板走 HTTPS 时才打开；局域网明文 HTTP 下必须保持注释， +# 否则浏览器丢弃 Cookie，表现为登录页反复刷新且无任何报错 +# MC_COOKIE_SECURE=1 +""", + encoding="utf-8", + ) + path.chmod(0o600) + + +async def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--host", default="192.168.2.27") + ap.add_argument("--port", type=int, default=3306) + ap.add_argument("--admin-user", required=True) + ap.add_argument("--admin-password", required=True) + ap.add_argument("--source-db", required=True) + ap.add_argument("--target-db", required=True) + ap.add_argument("--app-user", required=True) + ap.add_argument("--env-file", type=Path) + args = ap.parse_args() + + if args.source_db == args.target_db: + print("源库与目标库相同，拒绝执行", file=sys.stderr) + return 2 + + password = generate_password() + + await create_schema(args) + await provision_app_user(args, password) + await copy_tables(args) + + if args.env_file: + write_env(args.env_file, args, password) + print(f"[.env] 已写入 {args.env_file}（权限 600，密码未回显）") + + print("\n完成。") + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main()))