fix(db): _ensure_columns 改为按模型元数据推导,并补上漏加的调度字段
上一个提交加了 4 个调度字段,却没在 _ADDED_COLUMNS 里登记,后果是生产环境: - 任务列表接口报 Unknown column,UI 打不开任务列表 - 调度器每 20 秒 tick 一次炸一次,定时任务完全不会触发 最阴险的是启动完全正常——能连库、能起来,只是随后每条查询都失败。 - _ensure_columns 不再遍历手写清单,改为遍历 MonitorBase.metadata.sorted_tables, 从根上消掉「加了字段忘了登记」这类漏 - 新增 _column_ddl:用 CreateColumn 渲染类型与可空性,并给 NOT NULL 列补 DEFAULT。 模型的 default= 是 ORM 侧行为、不会进 DDL,而给已有数据的表加 NOT NULL 列必须有值, 否则能否成功取决于服务端 sql_mode - 主键列跳过:MySQL 不允许 AUTO_INCREMENT 与 DEFAULT 共存 新增 tests/test_monitor_column_migration.py 守住:每个 NOT NULL 列都必须能生成带 DEFAULT 的合法 ALTER。
This commit is contained in:
+63
-14
@@ -44,7 +44,9 @@ from contextlib import asynccontextmanager
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import AsyncIterator, Optional
|
from typing import AsyncIterator, Optional
|
||||||
|
|
||||||
from sqlalchemy import event, text
|
from sqlalchemy import Column, event, text
|
||||||
|
from sqlalchemy.dialects import mysql
|
||||||
|
from sqlalchemy.schema import CreateColumn
|
||||||
from sqlalchemy.ext.asyncio import (
|
from sqlalchemy.ext.asyncio import (
|
||||||
AsyncEngine,
|
AsyncEngine,
|
||||||
AsyncSession,
|
AsyncSession,
|
||||||
@@ -214,14 +216,53 @@ async def init_db() -> None:
|
|||||||
await _migrate_setting_keys(conn)
|
await _migrate_setting_keys(conn)
|
||||||
|
|
||||||
|
|
||||||
# Columns added to a table after it may already exist. ``create_all`` only
|
# ``create_all`` creates missing *tables* but never adds *columns* to a table that
|
||||||
# creates missing *tables*, so new columns need an explicit ALTER TABLE.
|
# already exists, so those need an explicit ALTER TABLE.
|
||||||
_ADDED_COLUMNS: dict[str, list[tuple[str, str]]] = {
|
#
|
||||||
"monitor_task": [
|
# Which columns those are is derived from the ORM metadata, not kept by hand. The
|
||||||
("notify_enabled", "BOOLEAN NOT NULL DEFAULT 0"),
|
# hand-kept version was a trap: forgetting to register a new column there still let
|
||||||
("last_notified_at", "BIGINT NULL"),
|
# the app start -- it connects fine, then fails on every query and every scheduler
|
||||||
],
|
# tick. Which is exactly what happened when the scheduling columns were added.
|
||||||
}
|
|
||||||
|
|
||||||
|
def _implicit_default(column: Column) -> Optional[str]:
|
||||||
|
"""A literal to seed existing rows with when a NOT NULL column is added."""
|
||||||
|
default = column.default
|
||||||
|
if default is not None and getattr(default, "is_scalar", False):
|
||||||
|
value = default.arg
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return "1" if value else "0"
|
||||||
|
if isinstance(value, (int, float)):
|
||||||
|
return str(value)
|
||||||
|
return "'" + str(value).replace("'", "''") + "'"
|
||||||
|
|
||||||
|
# No scalar default on the model. Fall back to the type's zero value, so that
|
||||||
|
# adding the column cannot depend on the server's sql_mode.
|
||||||
|
try:
|
||||||
|
python_type = column.type.python_type
|
||||||
|
except NotImplementedError:
|
||||||
|
return None
|
||||||
|
if python_type in (bool, int, float):
|
||||||
|
return "0"
|
||||||
|
if python_type is str:
|
||||||
|
return "''"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _column_ddl(column: Column) -> str:
|
||||||
|
"""One column as MySQL DDL for ``ALTER TABLE ... ADD COLUMN``.
|
||||||
|
|
||||||
|
``CreateColumn`` renders the name, type and nullability. The default is added
|
||||||
|
separately because a model's ``default=`` is applied by the ORM and never
|
||||||
|
reaches the DDL -- and a NOT NULL column added to a populated table needs a
|
||||||
|
value for the rows already sitting there.
|
||||||
|
"""
|
||||||
|
ddl = str(CreateColumn(column).compile(dialect=mysql.dialect()))
|
||||||
|
if not column.nullable and column.server_default is None:
|
||||||
|
seed = _implicit_default(column)
|
||||||
|
if seed is not None:
|
||||||
|
ddl += f" DEFAULT {seed}"
|
||||||
|
return ddl
|
||||||
|
|
||||||
|
|
||||||
async def _existing_columns(conn, table: str) -> set[str]:
|
async def _existing_columns(conn, table: str) -> set[str]:
|
||||||
@@ -240,14 +281,22 @@ async def _existing_columns(conn, table: str) -> set[str]:
|
|||||||
|
|
||||||
|
|
||||||
async def _ensure_columns(conn) -> None:
|
async def _ensure_columns(conn) -> None:
|
||||||
for table, columns in _ADDED_COLUMNS.items():
|
"""Add every model column the live table is missing."""
|
||||||
existing = await _existing_columns(conn, table)
|
for table in MonitorBase.metadata.sorted_tables:
|
||||||
|
existing = await _existing_columns(conn, table.name)
|
||||||
if not existing:
|
if not existing:
|
||||||
# Table did not exist before this run; create_all built it complete.
|
# Table did not exist before this run; create_all built it complete.
|
||||||
continue
|
continue
|
||||||
for name, ddl in columns:
|
for column in table.columns:
|
||||||
if name not in existing:
|
# Primary keys are always present, and MySQL rejects AUTO_INCREMENT
|
||||||
await conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {name} {ddl}"))
|
# alongside the DEFAULT this helper appends -- so skip them rather
|
||||||
|
# than emit DDL that could never run.
|
||||||
|
if column.name in existing or column.primary_key:
|
||||||
|
continue
|
||||||
|
print(f"[monitor.db] 补齐缺失字段 {table.name}.{column.name}", flush=True)
|
||||||
|
await conn.execute(
|
||||||
|
text(f"ALTER TABLE {table.name} ADD COLUMN {_column_ddl(column)}")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def _migrate_setting_keys(conn) -> None:
|
async def _migrate_setting_keys(conn) -> None:
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""Guards for the in-place column migration.
|
||||||
|
|
||||||
|
``create_all`` creates missing tables but never adds columns to a table that
|
||||||
|
already exists, so new model columns are applied by ``_ensure_columns``. That step
|
||||||
|
was originally driven by a hand-kept list, and forgetting to update it did not
|
||||||
|
fail loudly -- the app still started, connected, and then failed on every query
|
||||||
|
and every scheduler tick. These tests pin down its replacement, which derives the
|
||||||
|
work from the model metadata.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import Boolean, Column, Integer, MetaData, String, Table
|
||||||
|
|
||||||
|
from api.monitor import db
|
||||||
|
from api.monitor.models import MonitorBase
|
||||||
|
|
||||||
|
|
||||||
|
def _migratable_columns():
|
||||||
|
for table in MonitorBase.metadata.sorted_tables:
|
||||||
|
for column in table.columns:
|
||||||
|
if column.primary_key:
|
||||||
|
continue
|
||||||
|
yield pytest.param(column, id=f"{table.name}.{column.name}")
|
||||||
|
|
||||||
|
|
||||||
|
def _ddl(column: Column) -> str:
|
||||||
|
"""Render a detached column, so the tests never mutate the real metadata."""
|
||||||
|
scratch = Table("scratch", MetaData(), column)
|
||||||
|
return db._column_ddl(scratch.columns[0])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("column", _migratable_columns())
|
||||||
|
def test_column_renders_as_ddl(column):
|
||||||
|
ddl = db._column_ddl(column)
|
||||||
|
|
||||||
|
assert ddl.startswith(column.name)
|
||||||
|
# MySQL refuses AUTO_INCREMENT together with the DEFAULT this helper appends
|
||||||
|
# to NOT NULL columns.
|
||||||
|
assert "AUTO_INCREMENT" not in ddl
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("column", _migratable_columns())
|
||||||
|
def test_not_null_columns_carry_a_default(column):
|
||||||
|
"""So ADD COLUMN cannot fail on a table that already holds rows.
|
||||||
|
|
||||||
|
Without a DEFAULT, whether the ALTER succeeds depends on the server's
|
||||||
|
sql_mode -- not something a deployment should hinge on.
|
||||||
|
"""
|
||||||
|
if column.nullable:
|
||||||
|
pytest.skip("nullable column needs no seed value")
|
||||||
|
|
||||||
|
assert "DEFAULT" in db._column_ddl(column)
|
||||||
|
|
||||||
|
|
||||||
|
def test_boolean_default_becomes_a_mysql_literal():
|
||||||
|
"""Python's True is not a SQL keyword; it has to become 1."""
|
||||||
|
assert "DEFAULT 1" in _ddl(Column("flag", Boolean, nullable=False, default=True))
|
||||||
|
|
||||||
|
|
||||||
|
def test_string_defaults_are_quoted():
|
||||||
|
assert "DEFAULT 'interval'" in _ddl(
|
||||||
|
Column("mode", String(16), nullable=False, default="interval")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_integer_defaults_are_not_quoted():
|
||||||
|
ddl = _ddl(Column("n", Integer, nullable=False, default=0))
|
||||||
|
|
||||||
|
assert "DEFAULT 0" in ddl
|
||||||
|
assert "DEFAULT '0'" not in ddl
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_not_null_column_without_a_model_default_falls_back_to_zero():
|
||||||
|
"""Belt and braces: even a column the model gives no default still migrates."""
|
||||||
|
ddl = _ddl(Column("n", Integer, nullable=False))
|
||||||
|
|
||||||
|
assert "DEFAULT 0" in ddl
|
||||||
Reference in New Issue
Block a user