refactor: schema 收敛到 ORM metadata——5 张裸表升模型,方言无关的建表/补列/唯一索引
原来建表有三套来源(ORM create_all + 自建 SCHEMA_MIGRATIONS + agent_api 里的裸
CREATE TABLE)。这在 SQLite 下能活是因为 SQLite 的 DDL 极宽容;换 MySQL 后
agent_* 四张表的 `INTEGER PRIMARY KEY AUTOINCREMENT`、`TEXT DEFAULT ''`、
`TEXT PRIMARY KEY` 会全部建不出来,而失败被 `except: pass` 吞掉 —— 表现为
「经验库/动作库/会话功能静默失灵、日志里什么都看不到」。本期把它收敛成一套。
- core/models.py:
* 新增 5 个 ORM 模型:AppMeta / AgentExperience / ExperienceAudit /
AgentAction / AgentConversation(列名沿用历史,`app_meta.key` 在 MySQL 里是
保留字,ORM 属性名用 k,读写统一走 db_config.meta_get/meta_set)
* 新增 _long_text()(MySQL 用 MEDIUMTEXT,裸 TEXT 只有 64KB)与 _DOUBLE()
(评分别用单精度 FLOAT);_migrate_schema 不再执行 DDL,只维护版本账本,
SCHEMA_MIGRATIONS 的历史建表/加列条目 SQL 置 None
* 新增 _sync_columns():用 sqlalchemy.inspect 比对模型与实表补缺列,取代原来
靠 "duplicate column name" 报错文本判断的写法(换方言就失效)
* _ensure_unique_indexes() 按方言分叉:MySQL 5.7 没有过滤索引,改用
「虚拟生成列 + 唯一索引」复刻"空值不参与唯一约束"的语义
- web/agent_api.py: 删 4 段裸 CREATE TABLE;_ensure_* 收敛为 _ensure_tables()
(db.create_all 薄封装,失败必记日志);app_meta 读写与 INSERT OR IGNORE
改为方言中立
- core/device_discovery.py: app_meta 读写改用同一助手
- core/system_backup.py: SUMMARY_TABLES 由 db.metadata 派生 —— 备份覆盖红线
从"靠人记"变成结构上不可能漏;CURRENT_SCHEMA_VERSION 改从 models 导入
验证(均在 users.db 的一致快照副本上,不碰真实库):
空库 → 自建 12 张表 + 默认管理员 + 唯一索引 + schema_version=6;
老库 → 逐表行数与真实库完全一致,app_meta 键未变;
接口 → health/devices/jobs/groups/agent(经验库/动作库/会话) 全 200,
会话增删与备份导出(12 张表、无覆盖缺失)正常。
This commit is contained in:
+1
-1
@@ -129,7 +129,7 @@ def meta_set(key, value):
|
||||
"ON DUPLICATE KEY UPDATE value = VALUES(value)").format(k=k)
|
||||
else:
|
||||
sql = "INSERT OR REPLACE INTO app_meta ({k}, value) VALUES (:k, :v)".format(k=k)
|
||||
db.session.execute(text(sql), {"k": key, "v": value})
|
||||
db.session.execute(text(sql), {"k": key, "v": str(value)})
|
||||
db.session.commit()
|
||||
|
||||
|
||||
|
||||
@@ -61,9 +61,10 @@ def _fmt(ts=None):
|
||||
def get_settings():
|
||||
"""读发现配置(app_meta,缺键用 config 默认值补齐)。"""
|
||||
with _ctx():
|
||||
from core.db_config import meta_get
|
||||
|
||||
def _get(key, default):
|
||||
v = db.session.execute(
|
||||
db.text("SELECT value FROM app_meta WHERE key=:k"), {"k": key}).scalar()
|
||||
v = meta_get(key)
|
||||
return v if v is not None else default
|
||||
try:
|
||||
subnets = json.loads(_get(_K_SUBNETS, "[]")) or DISCOVERY_SUBNETS
|
||||
@@ -100,10 +101,8 @@ def save_settings(enabled=None, subnets=None, interval=None, port=None, auto_cla
|
||||
except (TypeError, ValueError):
|
||||
return False, "参数不合法"
|
||||
with _ctx():
|
||||
def _put(key, value):
|
||||
db.session.execute(
|
||||
db.text("INSERT OR REPLACE INTO app_meta(key,value) VALUES(:k,:v)"),
|
||||
{"k": key, "v": str(value)})
|
||||
from core.db_config import meta_set
|
||||
_put = meta_set # 方言中立 upsert(app_meta.key 在 MySQL 里是保留字)
|
||||
if enabled is not None:
|
||||
_put(_K_ENABLED, "1" if enabled else "0")
|
||||
if subnets is not None:
|
||||
|
||||
+233
-54
@@ -32,6 +32,22 @@ _log = get_logger("core.models")
|
||||
db = SQLAlchemy()
|
||||
|
||||
|
||||
def _long_text():
|
||||
"""长文本列:SQLite 用 TEXT,MySQL 用 MEDIUMTEXT。
|
||||
|
||||
裸 TEXT 在 MySQL 只有 64KB(而且是"字节"),AI 会话消息、任务参数这类
|
||||
JSON 文本很容易超;超了在严格模式下直接报错(不是截断)。
|
||||
"""
|
||||
from sqlalchemy.dialects.mysql import MEDIUMTEXT
|
||||
return db.Text().with_variant(MEDIUMTEXT(), "mysql")
|
||||
|
||||
|
||||
def _DOUBLE():
|
||||
"""REAL/DOUBLE:MySQL 的 FLOAT 是单精度,评分这类值会出现 7.8000001 这种尾巴。"""
|
||||
from sqlalchemy.dialects.mysql import DOUBLE
|
||||
return DOUBLE()
|
||||
|
||||
|
||||
@event.listens_for(Engine, "connect")
|
||||
def _sqlite_pragma(dbapi_connection, connection_record):
|
||||
"""SQLite 并发写优化:WAL 模式 + 忙等待超时 + 降同步级别。
|
||||
@@ -267,35 +283,108 @@ class PendingDevice(db.Model):
|
||||
"fingerprint": self.fingerprint or ""}
|
||||
|
||||
|
||||
# 版本化 schema 迁移:新增结构变更时在此追加 (版本号, 说明, SQL)
|
||||
# 版本号单调递增,只执行比当前 schema_version 新的迁移。
|
||||
class AppMeta(db.Model):
|
||||
"""全局 KV 配置(AI 配置、设备发现参数、schema_version、库环境标签…)。
|
||||
|
||||
⚠️ 列名沿用历史的 `key`/`value`:**`key` 在 MySQL 里是保留字**,
|
||||
因此不要直接拼裸 SQL 读写本表,统一走 `core/db_config.meta_get/meta_set`
|
||||
(方言中立、自动按方言加引号)。
|
||||
|
||||
ORM 属性名用 `k`:`key` 在 SQLAlchemy 声明式 API 里是保留名,不能直接当属性。
|
||||
"""
|
||||
__tablename__ = "app_meta"
|
||||
k = db.Column("key", db.String(64), primary_key=True)
|
||||
value = db.Column(db.Text)
|
||||
|
||||
def __repr__(self):
|
||||
return f"<AppMeta {self.k}>"
|
||||
|
||||
|
||||
class AgentExperience(db.Model):
|
||||
"""AI 控制台经验记忆:任务成功后的操作配方,下次相似任务检索注入。
|
||||
|
||||
recipe/tool_seq 存 JSON 文本;hits 是被召回引用的次数(巡检按它判重要性)。
|
||||
"""
|
||||
__tablename__ = "agent_experience"
|
||||
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
|
||||
task_prompt = db.Column(_long_text(), default="")
|
||||
recipe = db.Column(_long_text(), default="")
|
||||
tool_seq = db.Column(_long_text(), default="")
|
||||
hits = db.Column(db.Integer, default=0)
|
||||
created_at = db.Column(db.String(20), default="")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<AgentExperience {self.id}>"
|
||||
|
||||
|
||||
class ExperienceAudit(db.Model):
|
||||
"""经验巡检记录(AI 质检):疑似有问题的经验标 pending,删除只走人工确认。"""
|
||||
__tablename__ = "experience_audit"
|
||||
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
|
||||
exp_id = db.Column(db.Integer, nullable=False) # 对应的 agent_experience.id
|
||||
verdict = db.Column(db.String(20), default="") # keep / delete
|
||||
score = db.Column(db.Float().with_variant(_DOUBLE(), "mysql"), default=0)
|
||||
reason = db.Column(_long_text(), default="")
|
||||
hits = db.Column(db.Integer, default=0) # 巡检时的引用次数
|
||||
action = db.Column(db.String(20), default="pending") # pending/kept/deleted
|
||||
audited_at = db.Column(db.String(20), default="")
|
||||
|
||||
|
||||
class AgentAction(db.Model):
|
||||
"""动作库:带语义名的可复用动作单元(含元素定位、不含坐标)。
|
||||
|
||||
与任务级配方(agent_experience)互补;执行前按 name/别名/App 召回注入。
|
||||
"""
|
||||
__tablename__ = "agent_action"
|
||||
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
|
||||
name = db.Column(db.String(120), nullable=False)
|
||||
app = db.Column(db.String(80), default="")
|
||||
aliases = db.Column(_long_text(), default="[]")
|
||||
params = db.Column(_long_text(), default="[]")
|
||||
steps = db.Column(_long_text(), nullable=False)
|
||||
preconditions = db.Column(_long_text(), default="")
|
||||
hits = db.Column(db.Integer, default=0)
|
||||
source_prompt = db.Column(_long_text(), default="")
|
||||
created_at = db.Column(db.String(20), default="")
|
||||
updated_at = db.Column(db.String(20), default="")
|
||||
|
||||
|
||||
class AgentConversation(db.Model):
|
||||
"""AI 控制台会话:整个消息序列以 JSON 存在一行里(单会话几十 KB,够用)。"""
|
||||
__tablename__ = "agent_conversation"
|
||||
id = db.Column(db.String(20), primary_key=True)
|
||||
title = db.Column(db.String(100), default="")
|
||||
messages = db.Column(_long_text(), default="[]")
|
||||
created_at = db.Column(db.String(20), default="")
|
||||
updated_at = db.Column(db.String(20), default="")
|
||||
|
||||
|
||||
# 版本化 schema 迁移账本 (版本号, 说明, 数据回填 SQL 或 None)
|
||||
#
|
||||
# 注意:**建表与补列不再由这张表驱动**——改由 db.create_all() + _sync_columns()
|
||||
# 按模型定义自动完成(SQLite / MySQL 两种方言都正确)。这里只保留:
|
||||
# - 版本号(写进 app_meta.schema_version,备份/恢复时用来判断新旧)
|
||||
# - 未来可能需要的数据回填语句(纯数据操作,与方言无关)
|
||||
# 历史条目(v1~v6 建表/加列)保留在账本里以便追溯,SQL 位置一律为 None。
|
||||
SCHEMA_MIGRATIONS = [
|
||||
(1, "用户权限位:user 表新增 perms 列(JSON 数组,默认空=无业务权限,管理员不受限)",
|
||||
"ALTER TABLE user ADD COLUMN perms TEXT DEFAULT '[]'"),
|
||||
(2, "设备池:device 表(本地设备清单,替代 STF 池)",
|
||||
"CREATE TABLE IF NOT EXISTS device ("
|
||||
"serial VARCHAR(120) PRIMARY KEY,"
|
||||
"name VARCHAR(80) DEFAULT '',"
|
||||
"enabled BOOLEAN DEFAULT 1,"
|
||||
"note TEXT DEFAULT '',"
|
||||
"created_at VARCHAR(20) DEFAULT '')"),
|
||||
(3, "设备池:device 表新增 model 列(型号,在线时自动采集)",
|
||||
"ALTER TABLE device ADD COLUMN model TEXT DEFAULT ''"),
|
||||
(4, "自动发现:pending_device 待连接池表(扫描发现的设备,用户确认后才入正式池)",
|
||||
"CREATE TABLE IF NOT EXISTS pending_device ("
|
||||
"serial VARCHAR(120) PRIMARY KEY,"
|
||||
"source VARCHAR(20) DEFAULT '',"
|
||||
"first_seen VARCHAR(20) DEFAULT '',"
|
||||
"last_seen VARCHAR(20) DEFAULT '')"),
|
||||
(5, "设备池:device 表新增 fingerprint 列(设备指纹 ro.serialno,换 IP 后认领回原记录)",
|
||||
"ALTER TABLE device ADD COLUMN fingerprint VARCHAR(120) DEFAULT ''"),
|
||||
(6, "自动发现:pending_device 表新增 fingerprint 列(扫描时读取,用于提示是已有设备换了 IP)",
|
||||
"ALTER TABLE pending_device ADD COLUMN fingerprint VARCHAR(120) DEFAULT ''"),
|
||||
(1, "用户权限位:user 表新增 perms 列(JSON 数组,默认空=无业务权限,管理员不受限)", None),
|
||||
(2, "设备池:device 表(本地设备清单,替代 STF 池)", None),
|
||||
(3, "设备池:device 表新增 model 列(型号,在线时自动采集)", None),
|
||||
(4, "自动发现:pending_device 待连接池表(扫描发现的设备,用户确认后才入正式池)", None),
|
||||
(5, "设备池:device 表新增 fingerprint 列(设备指纹 ro.serialno,换 IP 后认领回原记录)", None),
|
||||
(6, "自动发现:pending_device 表新增 fingerprint 列(扫描时读取,用于提示是已有设备换了 IP)", None),
|
||||
]
|
||||
|
||||
# 唯一索引(部分索引:空值不参与唯一约束,兼容历史未命名/未采指纹的老数据)
|
||||
# 当前 schema 版本(备份/恢复用它判断新旧,也写进 app_meta.schema_version)
|
||||
CURRENT_SCHEMA_VERSION = max(v for v, _, _ in SCHEMA_MIGRATIONS)
|
||||
|
||||
# 唯一索引(语义 = 部分索引:空值不参与唯一约束,兼容历史未命名/未采指纹的老数据)
|
||||
# 名称唯一 = 设备的人可读标识;指纹唯一 = 一台物理设备在池中只能有一条记录
|
||||
_UNIQUE_INDEXES = (
|
||||
#
|
||||
# SQLite 支持带 WHERE 的部分索引,直接写;MySQL 5.7 不支持过滤索引,
|
||||
# 改用「虚拟生成列 + 唯一索引」复刻同一语义(唯一索引允许多个 NULL)——
|
||||
# 见 _ensure_unique_indexes() 的 MySQL 分支。
|
||||
_UNIQUE_INDEXES_SQLITE = (
|
||||
("ux_device_name", "CREATE UNIQUE INDEX IF NOT EXISTS ux_device_name "
|
||||
"ON device(name) WHERE name IS NOT NULL AND name <> ''"),
|
||||
("ux_device_fingerprint", "CREATE UNIQUE INDEX IF NOT EXISTS ux_device_fingerprint "
|
||||
@@ -303,66 +392,118 @@ _UNIQUE_INDEXES = (
|
||||
"AND fingerprint <> ''"),
|
||||
)
|
||||
|
||||
# MySQL:为空值生成 NULL 的伴随列,唯一索引建在它上面
|
||||
_MYSQL_UQ_COLUMNS = (
|
||||
# (表, 源列, 生成列名, 生成列类型)
|
||||
("device", "name", "name_uq", "VARCHAR(80)"),
|
||||
("device", "fingerprint", "fingerprint_uq", "VARCHAR(120)"),
|
||||
)
|
||||
_MYSQL_UNIQUE_INDEXES = (
|
||||
("ux_device_name", "device", "name_uq"),
|
||||
("ux_device_fingerprint", "device", "fingerprint_uq"),
|
||||
)
|
||||
|
||||
|
||||
def init_db(app):
|
||||
"""在 Flask app context 里初始化数据库 + 创建默认管理员。
|
||||
|
||||
web_server 启动时调用。自动迁移旧 groups.json/jobs.json 到 SQLite。
|
||||
web_server 启动时调用。自动迁移旧 groups.json/jobs.json 到数据库。
|
||||
|
||||
顺序:建表(create_all)→ 补列(_sync_columns)→ 版本账本 → 唯一索引 → 管理员/旧 JSON。
|
||||
建表与补列都以模型定义为准,SQLite / MySQL 两种方言自动匹配,不再手写 DDL。
|
||||
"""
|
||||
db.init_app(app)
|
||||
with app.app_context():
|
||||
db.create_all()
|
||||
_sync_columns()
|
||||
_migrate_schema()
|
||||
_ensure_unique_indexes()
|
||||
_ensure_default_admin()
|
||||
_migrate_old_json()
|
||||
|
||||
|
||||
def _migrate_schema():
|
||||
"""按 SCHEMA_MIGRATIONS 顺序执行版本化迁移,记录当前 schema_version。
|
||||
def _sync_columns():
|
||||
"""按模型定义补齐"实表缺的列"(幂等)。
|
||||
|
||||
create_all 只负责首次建表;结构变更必须走迁移,避免改了模型后老库对不上。
|
||||
取代原来靠 "duplicate column name" 报错文本判断的老写法:那是 SQLite 时代的
|
||||
容错路径,换 MySQL 后报错文本/错误码都不同,靠字符串匹配太脆。
|
||||
现在直接读数据库元数据(sqlalchemy.inspect)比对,缺什么补什么。
|
||||
"""
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.schema import CreateColumn
|
||||
|
||||
try:
|
||||
db.session.execute(text(
|
||||
"CREATE TABLE IF NOT EXISTS app_meta (key TEXT PRIMARY KEY, value TEXT)"))
|
||||
insp = inspect(db.engine)
|
||||
existing = set(insp.get_table_names())
|
||||
except Exception as e:
|
||||
_log.error(f"读取表结构失败,跳过补列: {e}")
|
||||
return
|
||||
|
||||
added = []
|
||||
for table in db.metadata.sorted_tables:
|
||||
if table.name not in existing:
|
||||
continue # 缺整表交给 create_all
|
||||
try:
|
||||
have = {c["name"] for c in insp.get_columns(table.name)}
|
||||
except Exception as e:
|
||||
_log.warning(f"读取 {table.name} 列失败: {e}")
|
||||
continue
|
||||
for col in table.columns:
|
||||
if col.name in have:
|
||||
continue
|
||||
try:
|
||||
ddl = CreateColumn(col).compile(dialect=db.engine.dialect)
|
||||
db.session.execute(
|
||||
text("ALTER TABLE {} ADD COLUMN {}".format(table.name, ddl)))
|
||||
db.session.commit()
|
||||
cur = db.session.execute(
|
||||
text("SELECT value FROM app_meta WHERE key='schema_version'")).scalar()
|
||||
current = int(cur) if cur else 0
|
||||
for version, desc, sql in SCHEMA_MIGRATIONS:
|
||||
added.append(f"{table.name}.{col.name}")
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
_log.warning(f"补列失败 {table.name}.{col.name}: {e}")
|
||||
if added:
|
||||
_log.info("schema 补列: " + ", ".join(added))
|
||||
|
||||
|
||||
def _migrate_schema():
|
||||
"""维护 schema_version 账本 + 执行数据回填类迁移。
|
||||
|
||||
建表/加列已由 create_all + _sync_columns 按模型自动完成,本函数不再写 DDL;
|
||||
SCHEMA_MIGRATIONS 里带 SQL 的条目只允许是方言无关的数据操作。
|
||||
"""
|
||||
from core import db_config
|
||||
try:
|
||||
try:
|
||||
current = int(db_config.meta_get("schema_version") or 0)
|
||||
except Exception:
|
||||
current = 0
|
||||
for version, desc, backfill_sql in SCHEMA_MIGRATIONS:
|
||||
if version <= current:
|
||||
continue
|
||||
if sql:
|
||||
if backfill_sql:
|
||||
try:
|
||||
db.session.execute(text(sql))
|
||||
except Exception as e:
|
||||
# 幂等兜底:目标结构已存在也视为该迁移生效。典型场景——
|
||||
# create_all 已按当前模型把列/表直接建好(如 perms、device.model),
|
||||
# 而 schema_version 又因历史中断没记录,导致每次启动重复 ALTER 报错。
|
||||
# duplicate column name 说明列已存在=迁移目标已达成:回滚本次语句后
|
||||
# 仍记录版本号,一次启动即自愈;其它异常才中止本批迁移。
|
||||
if "duplicate column name" not in str(e).lower():
|
||||
raise
|
||||
db.session.rollback()
|
||||
_log.info(f"schema 迁移 {version} 目标已存在,跳过: {desc}")
|
||||
db.session.execute(
|
||||
text("INSERT OR REPLACE INTO app_meta(key,value) VALUES('schema_version',:v)"),
|
||||
{"v": str(version)})
|
||||
db.session.execute(text(backfill_sql))
|
||||
db.session.commit()
|
||||
_log.info(f"schema 迁移到版本 {version}: {desc}")
|
||||
_ensure_unique_indexes()
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
_log.error(f"迁移 v{version} 数据回填失败: {e}")
|
||||
_log.info(f"schema 迁移到版本 {version}(结构由模型自动同步): {desc}")
|
||||
if current < CURRENT_SCHEMA_VERSION:
|
||||
db_config.meta_set("schema_version", str(CURRENT_SCHEMA_VERSION))
|
||||
except Exception as e:
|
||||
_log.error(f"schema 迁移失败(不阻塞启动): {e}")
|
||||
|
||||
|
||||
def _ensure_unique_indexes():
|
||||
"""建设备池的唯一索引(幂等)。
|
||||
"""建"空值不参与唯一约束"的设备名/指纹唯一索引(幂等,方言分叉)。
|
||||
|
||||
单独抽出来是因为索引不属于某个版本迁移:老库升级后也要补建。
|
||||
历史数据若存在重复(名称/指纹撞车),建索引会失败——只告警不回滚、
|
||||
不阻塞启动,由管理页提示用户改名(唯一约束从此刻起对新数据生效)。
|
||||
"""
|
||||
for name, sql in _UNIQUE_INDEXES:
|
||||
if db.engine.dialect.name == "mysql":
|
||||
_ensure_unique_indexes_mysql()
|
||||
return
|
||||
for name, sql in _UNIQUE_INDEXES_SQLITE:
|
||||
try:
|
||||
db.session.execute(text(sql))
|
||||
db.session.commit()
|
||||
@@ -371,6 +512,44 @@ def _ensure_unique_indexes():
|
||||
_log.warning(f"唯一索引 {name} 创建失败(历史数据可能有重复): {e}")
|
||||
|
||||
|
||||
def _ensure_unique_indexes_mysql():
|
||||
"""MySQL 5.7 没有过滤索引,用「虚拟生成列 + 唯一索引」复刻部分索引语义。
|
||||
|
||||
生成列把空值映射成 NULL,而唯一索引允许多个 NULL —— 正好等于
|
||||
「空值不参与唯一约束」。生成列不进 ORM 模型(进了 create_all 会尝试写入它
|
||||
并报 Error 3105),所以只能在这里用 DDL 补。
|
||||
"""
|
||||
from sqlalchemy import inspect
|
||||
coll = "utf8mb4_bin" # 与库默认一致:逐码点比较,等价 SQLite 的大小写敏感
|
||||
for table, col, gen, typ in _MYSQL_UQ_COLUMNS:
|
||||
try:
|
||||
have = {c["name"] for c in inspect(db.engine).get_columns(table)}
|
||||
if gen in have:
|
||||
continue
|
||||
db.session.execute(text(
|
||||
"ALTER TABLE {t} ADD COLUMN {g} {ty} COLLATE {coll} "
|
||||
"GENERATED ALWAYS AS (IF({c} IS NULL OR {c} = '', NULL, {c})) VIRTUAL"
|
||||
.format(t=table, g=gen, ty=typ, coll=coll, c=col)))
|
||||
db.session.commit()
|
||||
_log.info(f"已建生成列 {table}.{gen}(部分唯一索引的 MySQL 替代)")
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
_log.warning(f"生成列 {table}.{gen} 创建失败: {e}")
|
||||
|
||||
for name, table, gen in _MYSQL_UNIQUE_INDEXES:
|
||||
try:
|
||||
have = {i["name"] for i in inspect(db.engine).get_indexes(table)}
|
||||
if name in have:
|
||||
continue
|
||||
db.session.execute(
|
||||
text("CREATE UNIQUE INDEX {} ON {}({})".format(name, table, gen)))
|
||||
db.session.commit()
|
||||
_log.info(f"已建唯一索引 {name}")
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
_log.warning(f"唯一索引 {name} 创建失败(历史数据可能有重复): {e}")
|
||||
|
||||
|
||||
def _ensure_default_admin():
|
||||
"""首次启动创建默认管理员 admin/admin123。"""
|
||||
if not User.query.filter_by(username="admin").first():
|
||||
|
||||
+6
-13
@@ -26,26 +26,19 @@ from urllib.parse import quote
|
||||
from config import (DATA_DIR, APK_DIR, BACKUP_DIR,
|
||||
RESTORE_STAGING_DIR, RESTORE_PENDING_DIR)
|
||||
from core.logger import get_logger
|
||||
from core.models import SCHEMA_MIGRATIONS
|
||||
from core.models import CURRENT_SCHEMA_VERSION, db
|
||||
|
||||
_log = get_logger("core.backup")
|
||||
|
||||
# 当前代码期望的 schema 版本(来自版本化迁移表)
|
||||
CURRENT_SCHEMA_VERSION = max(v[0] for v in SCHEMA_MIGRATIONS) if SCHEMA_MIGRATIONS else 0
|
||||
|
||||
DB_FILE = os.path.join(DATA_DIR, "users.db")
|
||||
|
||||
# 判定「本平台备份库」的必需表(缺任何一张即拒绝导入)
|
||||
REQUIRED_TABLES = ("app_meta", "user", "task_job", "device_group")
|
||||
# 预览/清单里展示行数的业务表(存在才统计)
|
||||
# ⚠️ 红线:**新增任何持久化表,必须同步登记到这里**——备份覆盖清单以此为准,
|
||||
# 漏登记的表在导出清单/预览里不可见(2026-09-10 动作库 agent_action 即因此
|
||||
# 被误判为"没有备份":数据其实在 users.db 快照里,只是清单没列)。
|
||||
SUMMARY_TABLES = (
|
||||
"app_meta", "user", "device_group", "task_job", "custom_action",
|
||||
"apk_file", "device", "pending_device",
|
||||
"agent_conversation", "agent_experience", "experience_audit", "agent_action",
|
||||
)
|
||||
# 备份覆盖清单:**由模型元数据派生**,不再手工维护。
|
||||
# 这样「新增持久化表必须登记进备份清单」这条红线从"靠人记"变成结构上不可能漏
|
||||
# (2026-09-10 动作库 agent_action 就漏过:数据其实在快照里,只是清单没列,
|
||||
# 导出预览里看不到那一行 → 被误判成"没备份")。
|
||||
SUMMARY_TABLES = tuple(sorted(t.name for t in db.metadata.tables.values()))
|
||||
TABLE_LABELS = {
|
||||
"app_meta": "系统配置(app_meta)",
|
||||
"user": "用户", "device_group": "设备分组", "task_job": "任务计划",
|
||||
|
||||
+31
-71
@@ -60,14 +60,14 @@ def set_app(app):
|
||||
|
||||
|
||||
def _meta_get(key):
|
||||
return db.session.execute(
|
||||
db.text("SELECT value FROM app_meta WHERE key=:k"), {"k": key}).scalar() or ""
|
||||
# 走方言中立助手(app_meta 的列名 key 在 MySQL 里是保留字,裸 SQL 会语法错)
|
||||
from core.db_config import meta_get
|
||||
return meta_get(key) or ""
|
||||
|
||||
|
||||
def _meta_put(key, value):
|
||||
db.session.execute(
|
||||
db.text("INSERT OR REPLACE INTO app_meta(key,value) VALUES(:k,:v)"),
|
||||
{"k": key, "v": str(value)})
|
||||
from core.db_config import meta_set
|
||||
meta_set(key, str(value))
|
||||
|
||||
|
||||
def _read_cfg():
|
||||
@@ -75,29 +75,10 @@ def _read_cfg():
|
||||
|
||||
|
||||
# ================== 经验记忆(自进化) ==================
|
||||
# agent_experience:任务成功后的操作配方,下次相似任务检索注入 system prompt。
|
||||
# 原始 SQLite(CREATE IF NOT EXISTS 幂等),不进模型层迁移。
|
||||
_EXP_TABLE = """
|
||||
CREATE TABLE IF NOT EXISTS agent_experience (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
task_prompt TEXT DEFAULT '',
|
||||
recipe TEXT DEFAULT '',
|
||||
tool_seq TEXT DEFAULT '',
|
||||
hits INTEGER DEFAULT 0,
|
||||
created_at VARCHAR(20) DEFAULT '')"""
|
||||
|
||||
# 经验巡检(AI 质检):每日定时把经验交给模型评审,疑似问题标 pending 待人工确认。
|
||||
# 删除只允许人工(前端 confirm / 本 API),巡检绝不自动删。
|
||||
_AUDIT_TABLE = """
|
||||
CREATE TABLE IF NOT EXISTS experience_audit (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
exp_id INTEGER NOT NULL,
|
||||
verdict TEXT DEFAULT '', -- keep 保留 / delete 建议删除
|
||||
score REAL DEFAULT 0, -- 0-10 可用性评分
|
||||
reason TEXT DEFAULT '', -- 模型给出的理由
|
||||
hits INTEGER DEFAULT 0, -- 巡检时的引用次数(hits>0 被删要更谨慎)
|
||||
action TEXT DEFAULT 'pending', -- pending 待人工确认 / kept 已保留 / deleted 已删除
|
||||
audited_at VARCHAR(20) DEFAULT '')"""
|
||||
# agent_experience / experience_audit / agent_action / agent_conversation 四张表
|
||||
# 已在 core/models.py 里定义为 ORM 模型(唯一真相),建表统一由 db.create_all() 完成。
|
||||
# 以前这里是各写一段 SQLite 的 CREATE TABLE + try/except: pass —— 换 MySQL 后
|
||||
# AUTOINCREMENT / TEXT DEFAULT 全都建不出来,而异常被吞掉,表现为"功能静默失灵"。""
|
||||
|
||||
# 巡检评审提示词:模型只判「保留/建议删除」+ 评分 + 理由,不直接删。
|
||||
_AUDIT_PROMPT = """你是「手机自动化经验库」质检员。经验库存放 AI 成功操作手机后提炼的
|
||||
@@ -119,12 +100,20 @@ _AUDIT_PROMPT = """你是「手机自动化经验库」质检员。经验库存
|
||||
只输出 JSON:{{"keep": true或false, "score": 0到10的整数, "reason": "一句话中文理由"}}"""
|
||||
|
||||
|
||||
def _ensure_exp_table():
|
||||
def _ensure_tables():
|
||||
"""确保 agent 系四张表存在(幂等)。create_all 只建缺失的表,开销可以忽略。
|
||||
|
||||
失败必须落日志:以前是 except: pass,MySQL 下建表失败会被完全吞掉,
|
||||
表不存在 → 后续查询全报错,而日志里什么都看不到。
|
||||
"""
|
||||
try:
|
||||
db.session.execute(db.text(_EXP_TABLE))
|
||||
db.session.commit()
|
||||
except Exception:
|
||||
pass
|
||||
db.create_all()
|
||||
except Exception as e:
|
||||
_log.error(f"agent 表创建失败: {e}")
|
||||
|
||||
|
||||
def _ensure_exp_table():
|
||||
_ensure_tables()
|
||||
|
||||
|
||||
def _bigrams(text):
|
||||
@@ -366,19 +355,7 @@ def _save_experience(prompt, recipe, tool_seq):
|
||||
# steps 直接用编辑器 schema(open_app/click/input_text…),带**元素定位**
|
||||
# (selector_type/selector_value),**不含坐标**(分辨率/旋转/改版即失效)。
|
||||
# 复用:执行前按 name/别名/App 召回并注入 system prompt,模型可跳过重新探索。
|
||||
_ACTION_TABLE = (
|
||||
"CREATE TABLE IF NOT EXISTS agent_action ("
|
||||
"id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||||
"name TEXT NOT NULL,"
|
||||
"app TEXT DEFAULT '',"
|
||||
"aliases TEXT DEFAULT '[]',"
|
||||
"params TEXT DEFAULT '[]',"
|
||||
"steps TEXT NOT NULL,"
|
||||
"preconditions TEXT DEFAULT '',"
|
||||
"hits INTEGER DEFAULT 0,"
|
||||
"source_prompt TEXT DEFAULT '',"
|
||||
"created_at TEXT DEFAULT '',"
|
||||
"updated_at TEXT DEFAULT '')")
|
||||
# 建表见 core/models.py 的 AgentAction
|
||||
|
||||
# 允许沉淀的步骤类型(编辑器 STEP_TYPES 子集;显式排除 click_xy 等坐标类)
|
||||
_ACTION_STEP_TYPES = {
|
||||
@@ -398,11 +375,7 @@ _ACTION_TOOLS = {
|
||||
|
||||
|
||||
def _ensure_action_table():
|
||||
try:
|
||||
db.session.execute(db.text(_ACTION_TABLE))
|
||||
db.session.commit()
|
||||
except Exception:
|
||||
pass
|
||||
_ensure_tables()
|
||||
|
||||
|
||||
def _brief_result(result):
|
||||
@@ -731,11 +704,7 @@ _audit_state = {"running": False, "last": "", "last_summary": ""}
|
||||
|
||||
|
||||
def _ensure_audit_table():
|
||||
try:
|
||||
db.session.execute(db.text(_AUDIT_TABLE))
|
||||
db.session.commit()
|
||||
except Exception:
|
||||
pass
|
||||
_ensure_tables()
|
||||
|
||||
|
||||
def _review_one_exp(cfg, prompt, recipe, tool_seq, hits):
|
||||
@@ -1002,22 +971,11 @@ def agent_actions_save():
|
||||
|
||||
# ================== 历史会话(DeepSeek 式:多会话持久化) ==================
|
||||
# 会话 = 消息序列(JSON),一轮 run 的 user/assistant 文本完成后追加落库。
|
||||
# 单表 JSON 存储(每会话几十 KB 内,无需独立消息表)。
|
||||
_CONV_TABLE = """
|
||||
CREATE TABLE IF NOT EXISTS agent_conversation (
|
||||
id VARCHAR(20) PRIMARY KEY,
|
||||
title VARCHAR(100) DEFAULT '',
|
||||
messages TEXT DEFAULT '[]',
|
||||
created_at VARCHAR(20) DEFAULT '',
|
||||
updated_at VARCHAR(20) DEFAULT '')"""
|
||||
# 单表 JSON 存储(每会话几十 KB 内,无需独立消息表)。建表见 core/models.py。
|
||||
|
||||
|
||||
def _ensure_conv_table():
|
||||
try:
|
||||
db.session.execute(db.text(_CONV_TABLE))
|
||||
db.session.commit()
|
||||
except Exception:
|
||||
pass
|
||||
_ensure_tables()
|
||||
|
||||
|
||||
def _conv_msgs(conv_id):
|
||||
@@ -1181,13 +1139,15 @@ def agent_run():
|
||||
try:
|
||||
from datetime import datetime as _dt2
|
||||
now = _dt2.now().strftime("%Y-%m-%d %H:%M")
|
||||
# 上面刚确认过会话不存在,普通 INSERT 即可;
|
||||
# 并发下撞主键就回滚忽略(原来用 SQLite 专有的 INSERT OR IGNORE)
|
||||
db.session.execute(db.text(
|
||||
"INSERT OR IGNORE INTO agent_conversation(id, messages, "
|
||||
"INSERT INTO agent_conversation(id, messages, "
|
||||
"created_at, updated_at) VALUES(:i, '[]', :c, :c)"),
|
||||
{"i": conv_id, "c": now})
|
||||
db.session.commit()
|
||||
except Exception:
|
||||
pass
|
||||
db.session.rollback()
|
||||
with _lock:
|
||||
if _run["state"] == "running":
|
||||
return jsonify({"ok": False, "error":
|
||||
|
||||
Reference in New Issue
Block a user