butubb
b98e6deac1
一、平台侧（账号 → 发布计划页） - 新表 video_plan（schema v9→v10）：账号×发布日期×编号 → 素材 + 标题 + 发布状态 + 分享链接； 状态机 pending/ready/pushing/publishing/done/failed/unknown/skipped（**failed 与 unknown 必须分开**： 推送阶段的失败可安全重试；碰过抖音之后的岔子只能算"结果未知"，绝不自动重发） - 素材上传：文件名 `手机号_日期_编号`（编号可省）解析配对；标题 txt `标题内容_手机号_日期_编号`； 内容寻址落盘 data/videos/YYYY-MM/（sha1 分块算，同名不存两份），**不进整库备份**但进 manifest 反查 - 新蓝图 web/video_plan_api.py：上传/时间线/统计/单条增删改/推送到手机/标记结果/裁决/链接导出 CSV/ 任务列表与一键新建、**就地编辑**（GET/PUT /tasks/<id>）、**一键推送**（POST /push_all，按设备分组、设备内串行） - 账号页拆子分栏（台账 / 发布计划）+ static/admin/release.js；清理 job（04:41 僵尸回收+过期行、04:47 素材文件） - 上传体积：MAX_CONTENT_LENGTH（默认 2GiB）+ 413 JSON + nginx client_max_body_size（修现有 APK 上传隐患） 二、任务侧（平台推素材，抖音流程你自己写） - 新步骤 push_release「推送发布视频」：原子占位 → adb push → **touch 改成"现在"** → 清旧目录同名副本 → 触发扫描并**按路径**校验相册索引 → 标题写进剪贴板；默认目录 /sdcard/DCIM/Camera - 新步骤 mark_release「标记发布结果」：回写 done/failed/unknown，成功时抓作品分享链接、删手机素材 - input_text 支持 text_source=release_title（自动取计划标题 + 回读校验）； if_el 的候选值来源新增 release（**本机当前发布计划**的抖音号/昵称，发布前校验"登的是不是要发的号"） - build_release_steps 骨架 15 步：⓪ 亮屏 → ① 打开抖音(等首页) → ② 点「我」→ ③ 等抖音号出现 → ④ 条件判断(账号) → then ⑤ 推送 ⑥⑦⑧⑨⑩⑪⑫ 抖音点击/填标题 → ⑬ 标记 / else 发通知跳过 三、修（推送这一路的检测机制） - **uiautomator2 3.x 的 d.shell() 返回 ShellResponse（tuple 子类）不是 str**：`'x' in resp` 恒 False、 `.strip()` 不存在 → "推上去的文件大小不对"每次都判失败（文件其实推上去了）、相册校验永远报没进、 删除确认永远判没删掉。新增 publish_flow._sh() 统一取 .output；大小改成解析 ls -l 的大小列 - **adb push 保留本地 mtime** → 推 3 天前上传的素材在按时间排序的相册里排不到最前， "点第一个 = 刚推的那个"不成立 → 推完 touch - 相册校验**按路径**比（MediaStore 的 _data 会把目录小写、/storage/emulated/0 ≡ /sdcard）， 只比文件名会被老目录的同名残留骗过去 - 屏幕没亮就启动抖音会永远停在启动页（UI 树为空）→ 后面"点我/等抖音号"必然 miss， 最后报成误导人的"账号不符" → 骨架第一步固定加「亮屏」，open_app 等「首页」出现 四、其它 - core/ledger.serial_of()：设备名 → 当前地址（设备换 IP 后快照是错的） - 通知事件 task.video.published / task.video.failed；备份清单加 video_plan 与素材统计 - 文档同步：DATA_MODEL §2.11 + schema v10、API（新接口与语义）、TASK_DEV §4.7 专章、 ARCHITECTURE（账号页子分栏/release.js/两个 job）、DEPLOY（表数/nginx）、NOTIFY、DEVELOPMENT、README
662 lines
28 KiB
Python
662 lines
28 KiB
Python
"""系统数据备份导出/导入（数据库 + APK 文件）。
|
||
|
||
**归档格式：zip（`users.db` + `manifest.json` + 可选 `apks/*.apk`）。**
|
||
SQLite 在这里的角色是**交换格式**，不是运行时数据库——这在迁到 MySQL 之后依然成立，
|
||
带来三个好处：
|
||
1. 前端三个接口的请求/响应结构一行都不用改
|
||
2. 校验逻辑（完整性 / 必需表 / schema 版本 / 未登记表反向自检）可以整体沿用
|
||
3. 2026-09-13 之前导出的老备份继续可导入；也不依赖任何外部二进制
|
||
（`sqlite3` 是 Python 标准库，python:slim 容器里现成）
|
||
|
||
导出：把**当前库**（MySQL 或 SQLite）整库读出来，写进一个临时 SQLite 文件 → zip。
|
||
MySQL 下把这条连接提到 REPEATABLE READ，保证所有表读的是同一时刻。
|
||
导入：上传(zip/db) → 暂存校验预览 → 应用（先自动导出一份当前库到
|
||
BACKUP_DIR/pre_restore_*.zip 作安全网，再把暂存归档移到 RESTORE_PENDING_DIR）
|
||
→ 下次启动 consume_pending_restore() 在**单个事务内**整库替换生效。
|
||
|
||
为什么仍然要重启生效：TaskManager / device_pool 把任务与设备**缓存在内存**里，
|
||
整库替换后内存副本全部失效。（SQLite 时代还有"Windows 无法替换被持有的文件"这层，
|
||
改用 MySQL 之后只剩内存态这一层。）
|
||
|
||
安全红线：全程只读现有库 / 只写自己目录下的文件；替换数据用单事务，失败回滚，
|
||
绝不留下半新半旧的库；绝不 kill-server、绝不 disconnect（与全项目一致）。
|
||
"""
|
||
import json
|
||
import os
|
||
import shutil
|
||
import sqlite3
|
||
import time
|
||
import uuid
|
||
import zipfile
|
||
from urllib.parse import quote
|
||
|
||
from sqlalchemy import create_engine, inspect, select, text
|
||
|
||
from config import (DATA_DIR, APK_DIR, BACKUP_DIR,
|
||
RESTORE_STAGING_DIR, RESTORE_PENDING_DIR)
|
||
from core.logger import get_logger
|
||
from core.models import CURRENT_SCHEMA_VERSION, db
|
||
from core import notifier
|
||
|
||
_log = get_logger("core.backup")
|
||
|
||
DB_FILE = os.path.join(DATA_DIR, "users.db")
|
||
|
||
# 判定「本平台备份库」的必需表（缺任何一张即拒绝导入）
|
||
REQUIRED_TABLES = ("app_meta", "user", "task_job", "device_group")
|
||
# 备份覆盖清单：**由模型元数据派生**，不再手工维护。
|
||
# 这样「新增持久化表必须登记进备份清单」这条红线从"靠人记"变成结构上不可能漏
|
||
# （2026-09-10 动作库 agent_action 就漏过：数据其实在快照里，只是清单没列，
|
||
# 导出预览里看不到那一行 → 被误判成"没备份"）。
|
||
SUMMARY_TABLES = tuple(sorted(t.name for t in db.metadata.tables.values()))
|
||
TABLE_LABELS = {
|
||
"app_meta": "系统配置(app_meta)",
|
||
"user": "用户", "device_group": "设备分组", "task_job": "任务计划",
|
||
"custom_action": "自定义动作", "apk_file": "APK 记录", "device": "设备池",
|
||
"pending_device": "待连接设备", "agent_conversation": "AI 会话",
|
||
"agent_experience": "经验库", "experience_audit": "经验巡检",
|
||
"agent_action": "动作库", "device_install_log": "设备端安装记录",
|
||
"task_step_log": "任务步骤明细",
|
||
"done_mark": "去重记录（已做过）",
|
||
"device_account": "账号台账",
|
||
"video_plan": "视频发布计划",
|
||
}
|
||
_STAGE_TTL = 1800 # 导入暂存有效期（秒）
|
||
|
||
|
||
class BackupError(Exception):
|
||
"""备份/导入相关可预期错误（msg 直接给前端展示）。"""
|
||
|
||
|
||
# ================== 通用工具 ==================
|
||
def _ts():
|
||
return time.strftime("%Y%m%d_%H%M%S")
|
||
|
||
|
||
def _sqlite_uri(path):
|
||
"""把路径转成 sqlite URI（兼容含空格/中文/反斜杠的 Windows 路径）。"""
|
||
return "file:{}".format(quote(os.path.abspath(path).replace("\\", "/"), safe="/:"))
|
||
|
||
|
||
def _readonly_uri(path):
|
||
"""sqlite 只读 URI。"""
|
||
return _sqlite_uri(path) + "?mode=ro"
|
||
|
||
|
||
def _sqlite_engine_url(path):
|
||
"""SQLAlchemy 用的 SQLite URL（注意与 file: URI 不同，engine 不吃那个格式）。"""
|
||
return "sqlite:///" + os.path.abspath(path).replace("\\", "/")
|
||
|
||
|
||
def _connect_readonly(path):
|
||
con = sqlite3.connect(_readonly_uri(path), uri=True)
|
||
con.text_factory = str # 与平台一致按 UTF-8 读（表内文本均为 UTF-8）
|
||
return con
|
||
|
||
|
||
def remove_quiet(path):
|
||
"""尽力删除文件/目录（发送完成清理等场景，忽略不存在）。"""
|
||
try:
|
||
if os.path.isdir(path):
|
||
shutil.rmtree(path, ignore_errors=True)
|
||
elif os.path.exists(path):
|
||
os.remove(path)
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def _ensure_dir(path):
|
||
os.makedirs(path, exist_ok=True)
|
||
|
||
|
||
def _list_tables(con):
|
||
rows = con.execute(
|
||
"SELECT name FROM sqlite_master WHERE type='table'").fetchall()
|
||
return {r[0] for r in rows}
|
||
|
||
|
||
def _read_schema_version(con):
|
||
try:
|
||
row = con.execute(
|
||
"SELECT value FROM app_meta WHERE key='schema_version'").fetchone()
|
||
return int(row[0]) if row and row[0] else 0
|
||
except Exception:
|
||
return 0
|
||
|
||
|
||
def _table_rows(con, table):
|
||
try:
|
||
return con.execute('SELECT COUNT(*) FROM "%s"' % table).fetchone()[0]
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _summary_info(con):
|
||
"""表行数摘要（只列出存在的表）。"""
|
||
rows = []
|
||
for t in SUMMARY_TABLES:
|
||
n = _table_rows(con, t)
|
||
if n is not None:
|
||
rows.append({"table": t, "label": TABLE_LABELS.get(t, t), "rows": n})
|
||
return rows
|
||
|
||
|
||
def _video_summary():
|
||
"""视频素材目录的概况（写进 manifest；**不进备份包**，但要交代清楚）。
|
||
|
||
· `missing`：`video_plan` 引用了、但文件已经不在了的清单 —— 反向自检
|
||
（与 `coverage_missing` 同一个思路：别让"看起来有其实没有"悄悄发生）
|
||
"""
|
||
try:
|
||
from config import VIDEO_DIR
|
||
except Exception:
|
||
return {"included": False, "error": "config 读取失败"}
|
||
count = 0
|
||
size = 0
|
||
try:
|
||
for root, _dirs, files in os.walk(VIDEO_DIR):
|
||
for fn in files:
|
||
if fn.startswith(".tmp_"):
|
||
continue
|
||
try:
|
||
size += os.path.getsize(os.path.join(root, fn))
|
||
count += 1
|
||
except OSError:
|
||
pass
|
||
except Exception:
|
||
pass
|
||
missing = []
|
||
try:
|
||
from core.models import VideoPlan
|
||
with db.app_context():
|
||
for r in VideoPlan.query.filter(VideoPlan.video_file != "").all():
|
||
p = os.path.join(VIDEO_DIR, r.video_file)
|
||
if not os.path.exists(p):
|
||
missing.append({"id": r.id, "file": r.video_file})
|
||
except Exception:
|
||
pass
|
||
return {"included": False, "dir": "data/videos", "count": count, "bytes": size,
|
||
"note": "视频素材不进备份包（体积）。恢复后需重新上传；"
|
||
"计划、发布状态与分享链接在 video_plan 表里，已备份。",
|
||
"missing": missing[:200], "missing_count": len(missing)}
|
||
|
||
|
||
def _prune_old_exports(max_age=3600):
|
||
"""清理过期的导出临时 zip（下载完成后的 call_on_close 在 Windows 上可能
|
||
因文件锁删不掉，这里按时间兜底清理；保留近 1 小时的便于失败重试）。"""
|
||
now = time.time()
|
||
if not os.path.isdir(BACKUP_DIR):
|
||
return
|
||
for n in os.listdir(BACKUP_DIR):
|
||
if n.startswith("export_") and n.endswith(".zip"):
|
||
p = os.path.join(BACKUP_DIR, n)
|
||
try:
|
||
if now - os.path.getmtime(p) > max_age:
|
||
os.remove(p)
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def _prune_stale_staging():
|
||
"""清理超时未应用的暂存目录（TTL 后自动删除）。"""
|
||
now = time.time()
|
||
if not os.path.isdir(RESTORE_STAGING_DIR):
|
||
return
|
||
for name in os.listdir(RESTORE_STAGING_DIR):
|
||
p = os.path.join(RESTORE_STAGING_DIR, name)
|
||
try:
|
||
if os.path.isdir(p) and now - os.path.getmtime(p) > _STAGE_TTL:
|
||
remove_quiet(p)
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def _current_env_label():
|
||
"""当前库登记的环境标签（dev/prod）与库唯一标识（未登记则空）。"""
|
||
from core import db_config
|
||
try:
|
||
return db_config.meta_get("deployment_env") or "", db_config.meta_get("deployment_id") or ""
|
||
except Exception:
|
||
return "", ""
|
||
|
||
|
||
# ================== 导出：当前库 → SQLite 归档 ==================
|
||
def dump_to_sqlite(dest_path):
|
||
"""把当前数据库整库导出成一个 SQLite 文件（备份归档格式）。
|
||
|
||
一致性：MySQL 下把这条连接提到 REPEATABLE READ，让所有表读同一个时间点
|
||
（应用全局是 READ COMMITTED，事务提交后自增/被改的行会"漂移"）。
|
||
SQLite 下不需要——单文件库天然一致。
|
||
"""
|
||
_ensure_dir(os.path.dirname(os.path.abspath(dest_path)))
|
||
remove_quiet(dest_path)
|
||
remove_quiet(dest_path + "-wal")
|
||
remove_quiet(dest_path + "-shm")
|
||
|
||
out_eng = create_engine(_sqlite_engine_url(dest_path))
|
||
is_mysql = db.engine.dialect.name == "mysql"
|
||
conn = db.engine.connect()
|
||
if is_mysql:
|
||
conn = conn.execution_options(isolation_level="REPEATABLE READ")
|
||
|
||
dumped = []
|
||
try:
|
||
# 归档库用同一套模型建表（mysql_* 表选项在 SQLite 下被忽略）
|
||
db.metadata.create_all(out_eng)
|
||
src_cols = {}
|
||
insp = inspect(conn)
|
||
for table in db.metadata.sorted_tables:
|
||
try:
|
||
src_cols[table.name] = {c["name"] for c in insp.get_columns(table.name)}
|
||
except Exception:
|
||
src_cols[table.name] = set()
|
||
|
||
with out_eng.begin() as out:
|
||
for table in db.metadata.sorted_tables:
|
||
have = src_cols.get(table.name) or set()
|
||
cols = [c for c in table.columns if c.name in have]
|
||
if not cols:
|
||
_log.warning(f"导出跳过 {table.name}：源库没有这张表")
|
||
continue
|
||
try:
|
||
rows = [dict(r._mapping)
|
||
for r in conn.execute(select(*cols))]
|
||
except Exception as e:
|
||
_log.error(f"导出读取 {table.name} 失败: {e}")
|
||
raise BackupError(f"导出失败：读取 {table.name} 出错（{e}）")
|
||
if rows:
|
||
out.execute(table.insert(), rows)
|
||
dumped.append((table.name, len(rows)))
|
||
finally:
|
||
conn.close()
|
||
out_eng.dispose()
|
||
|
||
# 归档库收尾：落回单文件 + 补"空值不参与唯一"的唯一索引。
|
||
# SQLAlchemy 建的 sqlite 连接会被 PRAGMA 监听器设成 WAL，留下 -wal/-shm；
|
||
# zip 里只打包主文件，所以必须先 checkpoint 回 DELETE 模式再删附属文件。
|
||
try:
|
||
con = sqlite3.connect(dest_path)
|
||
try:
|
||
con.execute("PRAGMA journal_mode=DELETE")
|
||
except sqlite3.Error as e:
|
||
_log.warning(f"归档库切回单文件模式失败: {e}")
|
||
for name, sql in (("ux_device_name",
|
||
"CREATE UNIQUE INDEX IF NOT EXISTS ux_device_name "
|
||
"ON device(name) WHERE name IS NOT NULL AND name <> ''"),
|
||
("ux_device_fingerprint",
|
||
"CREATE UNIQUE INDEX IF NOT EXISTS ux_device_fingerprint "
|
||
"ON device(fingerprint) WHERE fingerprint IS NOT NULL "
|
||
"AND fingerprint <> ''")):
|
||
try:
|
||
con.execute(sql)
|
||
except Exception as e:
|
||
_log.warning(f"归档库建索引 {name} 失败（数据可能有重复）: {e}")
|
||
con.commit()
|
||
con.close()
|
||
except Exception as e:
|
||
_log.warning(f"归档库收尾失败（不影响导出）: {e}")
|
||
remove_quiet(dest_path + "-wal")
|
||
remove_quiet(dest_path + "-shm")
|
||
|
||
_log.info("已导出到 SQLite 归档: %s（%d 张表）",
|
||
os.path.basename(dest_path), len(dumped))
|
||
return dumped
|
||
|
||
|
||
# 兼容旧名（历史代码/脚本里叫 snapshot_db）
|
||
snapshot_db = dump_to_sqlite
|
||
|
||
|
||
def create_export(include_apk=True):
|
||
"""生成导出 zip，返回 (zip_path, filename, manifest)。"""
|
||
_ensure_dir(BACKUP_DIR)
|
||
_prune_old_exports()
|
||
base = os.path.join(BACKUP_DIR, f"export_{_ts()}")
|
||
snap = base + ".snapshot.db"
|
||
dump_to_sqlite(snap)
|
||
|
||
apk_meta = []
|
||
if include_apk and os.path.isdir(APK_DIR):
|
||
for fname in sorted(os.listdir(APK_DIR)):
|
||
if fname.lower().endswith(".apk"):
|
||
fp = os.path.join(APK_DIR, fname)
|
||
try:
|
||
apk_meta.append({"name": fname, "size": os.path.getsize(fp)})
|
||
except OSError:
|
||
pass
|
||
|
||
env_label, db_id = _current_env_label()
|
||
zip_path = base + ".zip"
|
||
con = _connect_readonly(snap)
|
||
try:
|
||
manifest = {
|
||
"format": "auto_control_backup",
|
||
"version": 2,
|
||
"created_at": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||
"schema_version": _read_schema_version(con),
|
||
"db_backend": db.engine.dialect.name, # mysql / sqlite
|
||
"deployment_env": env_label, # 来源环境（跨环境导入时告警）
|
||
"source_db_id": db_id, # 来源库唯一标识
|
||
"include_apk": include_apk,
|
||
"tables": _summary_info(con),
|
||
"apks": apk_meta,
|
||
# 视频发布计划的素材**不进备份包**（几十 GB 会把备份能力搞坏），
|
||
# 但必须在 manifest 里交代清楚 —— 否则就是"以为备份了其实没有"（红线）。
|
||
"videos": _video_summary(),
|
||
}
|
||
# 覆盖自检：登记在册的业务表若在快照里缺失（新增功能忘了登记 / 建表失败），
|
||
# 显式告警并写进 manifest——避免"以为备份了其实没有"（备份覆盖红线）。
|
||
_present = {t["table"] for t in manifest["tables"]}
|
||
_missing = [t for t in SUMMARY_TABLES if t not in _present]
|
||
if _missing:
|
||
manifest["coverage_missing"] = _missing
|
||
_log.warning(f"备份覆盖检查：以下登记表未纳入快照，请确认是否应备份: {_missing}")
|
||
finally:
|
||
con.close()
|
||
|
||
try:
|
||
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as z:
|
||
z.write(snap, arcname="users.db")
|
||
for a in apk_meta:
|
||
z.write(os.path.join(APK_DIR, a["name"]), arcname="apks/" + a["name"])
|
||
z.writestr("manifest.json", json.dumps(
|
||
manifest, ensure_ascii=False, indent=2))
|
||
finally:
|
||
remove_quiet(snap)
|
||
return zip_path, os.path.basename(zip_path), manifest
|
||
|
||
|
||
# ================== 校验 ==================
|
||
def validate_backup(db_file):
|
||
"""校验备份库可读、是本平台库。返回 (ok, info|error_msg)。"""
|
||
if not os.path.exists(db_file):
|
||
return False, "缺少数据库文件 users.db"
|
||
try:
|
||
con = _connect_readonly(db_file)
|
||
except sqlite3.Error as e:
|
||
return False, f"无法打开数据库: {e}"
|
||
try:
|
||
integrity = con.execute("PRAGMA integrity_check").fetchone()[0]
|
||
if integrity != "ok":
|
||
return False, f"数据库完整性校验失败: {integrity}"
|
||
tables = _list_tables(con)
|
||
missing = [t for t in REQUIRED_TABLES if t not in tables]
|
||
if missing:
|
||
return False, f"不是本平台备份库（缺少必需表: {', '.join(missing)}）"
|
||
schema_version = _read_schema_version(con)
|
||
missing_optional = [t for t in SUMMARY_TABLES
|
||
if t not in REQUIRED_TABLES and t not in tables]
|
||
warnings = []
|
||
if schema_version < CURRENT_SCHEMA_VERSION:
|
||
warnings.append(
|
||
f"备份 schema 较旧（v{schema_version} < 当前 v{CURRENT_SCHEMA_VERSION}），"
|
||
f"缺失的表/列按当前结构补空，不会报错但那些数据回不来")
|
||
elif schema_version > CURRENT_SCHEMA_VERSION:
|
||
warnings.append(
|
||
f"备份 schema 较新（v{schema_version} > 当前 v{CURRENT_SCHEMA_VERSION}），"
|
||
f"当前平台版本可能读不了新增列/表，建议先升级再恢复")
|
||
if missing_optional:
|
||
warnings.append("备份缺少部分可选表（" + "、".join(
|
||
TABLE_LABELS.get(t, t) for t in missing_optional)
|
||
+ "），导入后这些表为空")
|
||
# 反向自检：备份里有"未登记"的表 → 提醒把它纳入覆盖清单（红线）
|
||
extra_tables = sorted(t for t in tables
|
||
if t not in SUMMARY_TABLES and not t.startswith("sqlite_"))
|
||
if extra_tables:
|
||
warnings.append("备份含未登记的其它表（" + "、".join(extra_tables)
|
||
+ "）——如属业务数据，请登记进 core/models.py 的模型与 "
|
||
"core/system_backup.py 的 TABLE_LABELS")
|
||
warnings.append("备份为全量数据：含用户口令哈希、AI 配置里的 API Key 等敏感信息，请妥善保管")
|
||
return True, {
|
||
"integrity": integrity,
|
||
"schema_version": schema_version,
|
||
"current_schema_version": CURRENT_SCHEMA_VERSION,
|
||
"tables": _summary_info(con),
|
||
"missing_optional": missing_optional,
|
||
"extra_tables": extra_tables,
|
||
"warnings": warnings,
|
||
}
|
||
except sqlite3.Error as e:
|
||
return False, f"读取数据库失败: {e}"
|
||
finally:
|
||
con.close()
|
||
|
||
|
||
# ================== 导入：暂存 + 预览 ==================
|
||
def _extract_zip(zip_path, staging_dir):
|
||
"""解压备份 zip：取顶层 users.db、可选 apks/*.apk 与 manifest.json（防路径穿越）。"""
|
||
db_dest = os.path.join(staging_dir, "users.db")
|
||
apk_dest = os.path.join(staging_dir, "apks")
|
||
got_db = False
|
||
with zipfile.ZipFile(zip_path) as z:
|
||
for info in z.infolist():
|
||
if info.is_dir():
|
||
continue
|
||
name = info.filename.replace("\\", "/")
|
||
base = os.path.basename(name)
|
||
if name == "users.db" and base == "users.db":
|
||
z.extract(info, staging_dir) # 已确认顶层名字，无穿越
|
||
got_db = True
|
||
elif name == "manifest.json":
|
||
with z.open(info) as src, open(
|
||
os.path.join(staging_dir, "manifest.json"), "wb") as dst:
|
||
shutil.copyfileobj(src, dst)
|
||
elif name.startswith("apks/") and base.lower().endswith(".apk"):
|
||
_ensure_dir(apk_dest)
|
||
with z.open(info) as src, open(
|
||
os.path.join(apk_dest, base), "wb") as dst:
|
||
shutil.copyfileobj(src, dst)
|
||
if not got_db:
|
||
raise BackupError("zip 内未找到 users.db（顶层）")
|
||
return db_dest
|
||
|
||
|
||
def _read_staged_manifest(staging_dir):
|
||
p = os.path.join(staging_dir, "manifest.json")
|
||
if not os.path.exists(p):
|
||
return {}
|
||
try:
|
||
with open(p, encoding="utf-8") as f:
|
||
return json.load(f) or {}
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def stage_upload(file_storage):
|
||
"""保存上传备份并校验，返回 (token, info)。失败抛 BackupError。"""
|
||
_prune_stale_staging()
|
||
orig_name = file_storage.filename or "backup"
|
||
token = uuid.uuid4().hex[:12]
|
||
staging_dir = os.path.join(RESTORE_STAGING_DIR, token)
|
||
_ensure_dir(staging_dir)
|
||
low = (orig_name or "").lower()
|
||
try:
|
||
raw_path = os.path.join(staging_dir, "upload" + ("." + low.rsplit(".", 1)[1] if "." in low else ""))
|
||
file_storage.save(raw_path)
|
||
if low.endswith(".zip"):
|
||
_extract_zip(raw_path, staging_dir)
|
||
db_file = os.path.join(staging_dir, "users.db")
|
||
remove_quiet(raw_path)
|
||
elif low.endswith(".db"):
|
||
db_file = os.path.join(staging_dir, "users.db")
|
||
os.replace(raw_path, db_file)
|
||
else:
|
||
raise BackupError("仅支持 .zip（平台导出）或 .db（备份库文件）")
|
||
ok, info = validate_backup(db_file)
|
||
if not ok:
|
||
raise BackupError(info)
|
||
info["file_name"] = orig_name
|
||
info["file_size"] = os.path.getsize(db_file)
|
||
|
||
# 跨环境提示（防混库第 5 层）：备份来自哪个环境、当前连的是哪个环境
|
||
man = _read_staged_manifest(staging_dir)
|
||
info["source_env"] = man.get("deployment_env") or ""
|
||
info["source_db_id"] = man.get("source_db_id") or ""
|
||
info["source_backend"] = man.get("db_backend") or "sqlite(旧版备份)"
|
||
cur_env, _ = _current_env_label()
|
||
info["current_env"] = cur_env
|
||
if info["source_env"] and cur_env and info["source_env"] != cur_env:
|
||
info["warnings"].append(
|
||
f"该备份来自 {info['source_env']} 环境，而当前连接的是 {cur_env} 库 —— "
|
||
f"导入会覆盖当前全部数据。确认这是你要的操作再继续（跨环境导入默认被拒绝）。")
|
||
# 暂存目录里留住 manifest，apply 时还要用它做环境校验
|
||
return token, info
|
||
except BackupError:
|
||
remove_quiet(staging_dir)
|
||
raise
|
||
except Exception as e:
|
||
remove_quiet(staging_dir)
|
||
raise BackupError(f"暂存上传文件失败: {e}")
|
||
|
||
|
||
# ================== 导入：应用（落待生效任务） ==================
|
||
def apply_restore(token, force_env_mismatch=False):
|
||
"""校验 token → 自动备份当前库（zip 安全网）→ 把暂存归档移到 restore_pending。
|
||
|
||
返回 dict {ok, backup_name, message}；失败抛 BackupError。token 形如 12 位 hex。
|
||
"""
|
||
if not token or len(token) != 12 or not all(c in "0123456789abcdef" for c in token):
|
||
raise BackupError("无效的导入标识")
|
||
staging_dir = os.path.join(RESTORE_STAGING_DIR, token)
|
||
db_file = os.path.join(staging_dir, "users.db")
|
||
if not os.path.exists(db_file):
|
||
raise BackupError("导入会话已失效，请重新上传预览")
|
||
ok, info = validate_backup(db_file)
|
||
if not ok:
|
||
raise BackupError(info)
|
||
|
||
# 跨环境硬停：默认不许把 prod 的备份灌进 dev 库（反之亦然）
|
||
man = _read_staged_manifest(staging_dir)
|
||
src_env = man.get("deployment_env") or ""
|
||
cur_env, _ = _current_env_label()
|
||
if src_env and cur_env and src_env != cur_env and not force_env_mismatch:
|
||
raise BackupError(
|
||
f"备份来自 {src_env} 环境，当前库是 {cur_env} 环境 —— 已拒绝导入。\n"
|
||
f"如确需跨环境恢复，请在确认页勾选「允许跨环境导入」后重试。")
|
||
|
||
# 1) 自动备份当前库（安全网，可回滚；导出成 zip，能直接再导入回来）
|
||
_ensure_dir(BACKUP_DIR)
|
||
try:
|
||
zip_path, _, _ = create_export(include_apk=True)
|
||
backup_name = f"pre_restore_{_ts()}.zip"
|
||
shutil.move(zip_path, os.path.join(BACKUP_DIR, backup_name))
|
||
except Exception as e:
|
||
raise BackupError(f"导入前自动备份当前库失败，已中止导入: {e}")
|
||
|
||
# 2) 落待生效恢复任务（旧未消费任务被覆盖——上一份已无意义）
|
||
if os.path.isdir(RESTORE_PENDING_DIR):
|
||
remove_quiet(RESTORE_PENDING_DIR)
|
||
_ensure_dir(RESTORE_PENDING_DIR)
|
||
os.replace(db_file, os.path.join(RESTORE_PENDING_DIR, "users.db"))
|
||
for extra in ("manifest.json",):
|
||
src = os.path.join(staging_dir, extra)
|
||
if os.path.exists(src):
|
||
shutil.move(src, os.path.join(RESTORE_PENDING_DIR, extra))
|
||
apk_src = os.path.join(staging_dir, "apks")
|
||
if os.path.isdir(apk_src) and os.listdir(apk_src):
|
||
shutil.move(apk_src, os.path.join(RESTORE_PENDING_DIR, "apks"))
|
||
remove_quiet(staging_dir)
|
||
|
||
schema_version = info.get("schema_version", 0)
|
||
_log.info(f"导入已落恢复任务: schema v{schema_version}，"
|
||
f"重启 web_server 后生效；当前库已自动备份 {backup_name}")
|
||
return {
|
||
"ok": True,
|
||
"backup_name": backup_name,
|
||
"message": "恢复任务已生成：当前库已自动备份，重启 web_server 后即应用导入的数据",
|
||
}
|
||
|
||
|
||
# ================== 启动消费（init_db 之后、TaskManager 之前调用）==================
|
||
def _pending_db_path():
|
||
"""待生效恢复的归档路径（兼容旧版扁平布局）。"""
|
||
p = os.path.join(RESTORE_PENDING_DIR, "users.db")
|
||
return p if os.path.exists(p) else None
|
||
|
||
|
||
def consume_pending_restore():
|
||
"""把 restore_pending/users.db 的内容在**单个事务内**整库替换进来。
|
||
|
||
与 SQLite 时代的"换文件"不同：现在是 DELETE 全表 + 分块 INSERT，全程一个事务——
|
||
任何一步失败就 rollback，当前数据保持原样（比换文件更安全）。
|
||
|
||
必须在 init_db() 之后调用（表已建好、app context 可用）；失败不阻塞启动：
|
||
坏归档挪到 BACKUP_DIR/restore_failed_*/ 并继续用当前数据。
|
||
返回是否真的应用了恢复。
|
||
"""
|
||
pending_db = _pending_db_path()
|
||
if not pending_db:
|
||
return False
|
||
# 消费前轻量复验（防止意外损坏的文件把主库换掉）
|
||
ok, msg = validate_backup(pending_db)
|
||
if not ok:
|
||
fail_dir = os.path.join(BACKUP_DIR, f"restore_failed_{_ts()}")
|
||
_ensure_dir(fail_dir)
|
||
shutil.move(pending_db, os.path.join(fail_dir, "users.db"))
|
||
remove_quiet(RESTORE_PENDING_DIR)
|
||
_log.error(f"恢复任务校验失败，已搁置到 {fail_dir}，继续使用当前库: {msg}")
|
||
notifier.notify("system.backup.restore_failed", error=str(msg)[:200],
|
||
fail_dir=fail_dir)
|
||
return False
|
||
|
||
try:
|
||
applied = _apply_archive_to_db(pending_db)
|
||
except Exception as e:
|
||
_log.error(f"应用备份恢复失败，已回滚（当前数据未改动）: {e}，"
|
||
f"归档保留在 {RESTORE_PENDING_DIR} 供排查")
|
||
return False
|
||
|
||
# 合并 apks（覆盖同名）
|
||
apk_src = os.path.join(RESTORE_PENDING_DIR, "apks")
|
||
if os.path.isdir(apk_src):
|
||
_ensure_dir(APK_DIR)
|
||
for fname in os.listdir(apk_src):
|
||
src = os.path.join(apk_src, fname)
|
||
dst = os.path.join(APK_DIR, fname)
|
||
if os.path.exists(dst):
|
||
os.remove(dst)
|
||
shutil.move(src, dst)
|
||
remove_quiet(RESTORE_PENDING_DIR)
|
||
schema_version = msg.get("schema_version", "?") if isinstance(msg, dict) else "?"
|
||
_log.info(f"已应用备份恢复（单事务替换）: {applied}，schema v{schema_version}，apks 已合并")
|
||
# 通知在"下一次启动"才发得出去（恢复本身是重启生效的），文案里说明白
|
||
notifier.notify("system.backup.restored", applied_rows=applied,
|
||
schema_version=schema_version)
|
||
return True
|
||
|
||
|
||
def _apply_archive_to_db(db_file, chunk=500):
|
||
"""把 SQLite 归档里的数据整库写进当前数据库，单事务、失败回滚。
|
||
|
||
用 DELETE 而不是 TRUNCATE：TRUNCATE 是 DDL，会隐式提交，破坏原子性。
|
||
"""
|
||
con = _connect_readonly(db_file)
|
||
arch_tables = _list_tables(con)
|
||
stats = []
|
||
try:
|
||
# 单事务：MySQL/SQLite 都支持事务内 DELETE + INSERT
|
||
with db.engine.begin() as dst:
|
||
for table in reversed(db.metadata.sorted_tables):
|
||
dst.execute(table.delete())
|
||
for table in db.metadata.sorted_tables:
|
||
if table.name not in arch_tables:
|
||
_log.warning(f"归档里没有 {table.name}，该表导入后为空")
|
||
continue
|
||
have = {r[1] for r in con.execute(
|
||
'PRAGMA table_info("%s")' % table.name)}
|
||
cols = [c for c in table.columns if c.name in have]
|
||
if not cols:
|
||
_log.warning(f"归档表 {table.name} 没有可用列，跳过")
|
||
continue
|
||
sel = "SELECT {} FROM \"{}\"".format(
|
||
", ".join('"%s"' % c.name for c in cols), table.name)
|
||
rows = con.execute(sel).fetchall()
|
||
names = [c.name for c in cols]
|
||
for i in range(0, len(rows), chunk):
|
||
batch = [dict(zip(names, r)) for r in rows[i:i + chunk]]
|
||
dst.execute(table.insert(), batch)
|
||
stats.append(f"{table.name}={len(rows)}")
|
||
finally:
|
||
con.close()
|
||
return " ".join(stats)
|