checkpoint: 架构治理四件套——统一 async / 测试真 PG / Redis 降级可选 / DPB 扶正

- A1 业务层统一 async:gencode 表内省改 run_sync 异步(不再阻塞事件循环)、
  check_db 改 async_engine、dict_util 启动预热异步载入;psycopg 降为
  APScheduler SQLAlchemyJobStore 专用同步孤岛(注释标注)
- A2 测试真 PG 化:conftest 弃 SQLite + mock Redis,改连真实 PG16(dpb_test)+
  Redis(number_gen advisory lock 路径真被测);新增 docker/test-compose.yaml
  测试依赖栈(postgres:16:5433 + redis:7:6380);run_ci 前置探测 + 容器兜底,
  pytest -m pg 10 套正式 tc 全过(真实 PG16 + Redis)
- B1 Redis 启动不强依赖:redis_connect 失败降级启动,缓存类回源 DB、存储类
  (会话/AI 配置/调度)经 require_redis 守卫返回 503;真机 3 阶段降级测试
  PASS=11 FAIL=0(独立 Redis 6390 + 后端 8091,不动共享 dev 服务)
- C 扶正 DPB:后端横幅/日志(dpb.log)/README/pyproject/ai_factory agent 名、
  前端 package.json/署名注释/链接文案/deploy.sh/docker 注释全部去 fastapiadmin;
  grep backend/app + frontend/web/src 零残留
- 验证:全量回归 pass=72 fail=0
This commit is contained in:
34047007@qq.com
2026-08-07 09:06:19 +08:00
parent e937fc930c
commit 86126ea16f
64 changed files with 546 additions and 359 deletions
@@ -1,14 +1,13 @@
import asyncio
from collections.abc import Sequence
from typing import TYPE_CHECKING
from sqlalchemy import Inspector, inspect, select, text
from sqlalchemy import inspect, select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.config.setting import settings
from app.core.base_crud import CRUDBase
from app.core.base_schema import AuthSchema
from app.core.database import engine
from app.core.database import async_engine
from app.core.logger import logger
from app.utils.common_util import search_to_dict
@@ -25,6 +24,12 @@ if TYPE_CHECKING:
from sqlalchemy.engine.reflection import Inspector
async def _run_inspector(func, *args):
"""在异步引擎连接上以 run_sync 执行同步 Inspector 操作,避免阻塞事件循环。"""
async with async_engine.connect() as conn:
return await conn.run_sync(func, *args)
class GenTableCRUD(CRUDBase[GenTableModel, GenTableSchema, GenTableSchema]):
"""代码生成业务表模块数据库操作层"""
@@ -142,19 +147,21 @@ class GenTableCRUD(CRUDBase[GenTableModel, GenTableSchema, GenTableSchema]):
database_name = settings.DATABASE_NAME
database_type = settings.DATABASE_TYPE
inspector: Inspector = inspect(engine)
table_names = inspector.get_table_names()
def _introspect(sync_conn) -> list[tuple[str, str]]:
insp = inspect(sync_conn)
out = []
for table_name in insp.get_table_names():
try:
table_comment = insp.get_table_comment(table_name)
comment = table_comment.get("text", "") if isinstance(table_comment, dict) else table_comment
out.append((table_name, comment or ""))
except Exception as e:
logger.warning(f"获取表 {table_name} 的注释失败: {e}")
out.append((table_name, ""))
return out
dict_data = []
for table_name in table_names:
try:
table_comment = inspector.get_table_comment(table_name)
comment = table_comment.get("text", "") if isinstance(table_comment, dict) else table_comment
table_comment = comment or ""
except Exception as e:
logger.warning(f"获取表 {table_name} 的注释失败: {e}")
table_comment = ""
for table_name, table_comment in await _run_inspector(_introspect):
# 统一处理 search 为 None 的情况,避免重复判断
if search:
# 表名过滤:忽略大小写,支持模糊匹配
@@ -296,26 +303,30 @@ class GenTableCRUD(CRUDBase[GenTableModel, GenTableSchema, GenTableSchema]):
database_name = settings.DATABASE_NAME
database_type = settings.DATABASE_TYPE
inspector: Inspector = inspect(engine)
all_table_names = set(inspector.get_table_names())
def _introspect(sync_conn, names: list[str]) -> list[tuple[str, str]]:
insp = inspect(sync_conn)
all_table_names = set(insp.get_table_names())
out = []
for table_name in names:
if table_name not in all_table_names:
continue
try:
table_comment = insp.get_table_comment(table_name)
comment = table_comment.get("text", "") if isinstance(table_comment, dict) else (table_comment or "")
except Exception as e:
logger.warning(f"获取表 {table_name} 的注释失败: {e}")
comment = ""
out.append((table_name, comment or ""))
return out
results = []
for table_name in table_names:
if table_name not in all_table_names:
continue
try:
table_comment = inspector.get_table_comment(table_name)
comment = table_comment.get("text", "") if isinstance(table_comment, dict) else (table_comment or "")
except Exception as e:
logger.warning(f"获取表 {table_name} 的注释失败: {e}")
comment = ""
for table_name, comment in await _run_inspector(_introspect, table_names):
results.append(
GenDBTableSchema(
database_name=database_name,
table_name=table_name,
table_type=database_type,
table_comment=comment or "",
table_comment=comment,
),
)
@@ -330,8 +341,7 @@ class GenTableCRUD(CRUDBase[GenTableModel, GenTableSchema, GenTableSchema]):
返回:
- bool: 如果表存在返回True,否则返回False。
"""
inspector: Inspector = inspect(engine)
return inspector.has_table(table_name)
return await _run_inspector(lambda sync_conn: inspect(sync_conn).has_table(table_name))
async def get_db_table_comment(self, table_name: str) -> str:
"""获取数据库中指定表的注释(用于主子表场景下从库中加载子表元信息)。
@@ -342,16 +352,19 @@ class GenTableCRUD(CRUDBase[GenTableModel, GenTableSchema, GenTableSchema]):
返回:
- str: 表注释;表不存在或失败时为空字符串。
"""
inspector: Inspector = inspect(engine)
if not inspector.has_table(table_name):
return ""
try:
table_comment = inspector.get_table_comment(table_name)
comment = table_comment.get("text", "") if isinstance(table_comment, dict) else (table_comment or "")
return comment or ""
except Exception as e:
logger.warning(f"获取表 {table_name} 的注释失败: {e}")
return ""
def _get_comment(sync_conn) -> str:
insp = inspect(sync_conn)
if not insp.has_table(table_name):
return ""
try:
table_comment = insp.get_table_comment(table_name)
comment = table_comment.get("text", "") if isinstance(table_comment, dict) else (table_comment or "")
return comment or ""
except Exception as e:
logger.warning(f"获取表 {table_name} 的注释失败: {e}")
return ""
return await _run_inspector(_get_comment)
async def execute_sql(self, sql: str) -> bool:
"""执行SQL语句。
@@ -384,8 +397,8 @@ class GenTableColumnCRUD(CRUDBase[GenTableColumnModel, GenTableColumnSchema, Gen
super().__init__(model=GenTableColumnModel, auth=auth, db=db)
@staticmethod
def _sync_get_table_columns(database_type: str, table_name: str) -> list[dict]:
"""同步函数:获取数据库表的列信息
async def _get_table_columns(database_type: str, table_name: str) -> list[dict]:
"""在异步引擎连接上内省数据库表的列信息(不阻塞事件循环)。
参数:
- database_type: 数据库类型
@@ -394,62 +407,65 @@ class GenTableColumnCRUD(CRUDBase[GenTableColumnModel, GenTableColumnSchema, Gen
返回:
- list: 列信息列表
"""
# 使用SQLAlchemy Inspector获取表列信息
inspector: Inspector = inspect(engine)
def _introspect(sync_conn) -> list[dict]:
# 使用SQLAlchemy Inspector获取表列信息
inspector = inspect(sync_conn)
# 获取列信息
columns = inspector.get_columns(table_name)
# 获取列信息
columns = inspector.get_columns(table_name)
# 获取主键信息
try:
pk_constraint = inspector.get_pk_constraint(table_name)
primary_keys = set(pk_constraint.get("constrained_columns", [])) if pk_constraint else set()
except Exception:
primary_keys = set()
# 获取主键信息
try:
pk_constraint = inspector.get_pk_constraint(table_name)
primary_keys = set(pk_constraint.get("constrained_columns", [])) if pk_constraint else set()
except Exception:
primary_keys = set()
# 获取唯一约束信息
unique_columns = set()
unique_constraints = inspector.get_unique_constraints(table_name)
for constraint in unique_constraints:
unique_columns.update(constraint.get("column_names", []))
# 获取唯一约束信息
unique_columns = set()
unique_constraints = inspector.get_unique_constraints(table_name)
for constraint in unique_constraints:
unique_columns.update(constraint.get("column_names", []))
# 处理列信息
columns_list = []
for idx, column in enumerate(columns):
# 获取列的基本信息
column_name = column["name"]
column_type = str(column["type"])
is_nullable = column.get("nullable", True)
column_default = column.get("default", None)
# 获取列注释(如果有的话)
column_comment = column.get("comment", "")
# 判断是否为主键
is_pk = column_name in primary_keys
# 判断是否为唯一约束
is_unique = column_name in unique_columns
# 判断是否为自增列(基于数据库类型和列类型)
is_increment = column.get("autoincrement", False) in (True, "auto")
# 获取列长度(如果适用)
col_len = getattr(column["type"], "length", None)
column_length = str(col_len) if col_len is not None else ""
# 处理列信息
columns_list = []
for idx, column in enumerate(columns):
# 获取列的基本信息
column_name = column["name"]
column_type = str(column["type"])
is_nullable = column.get("nullable", True)
column_default = column.get("default", None)
# 获取列注释(如果有的话)
column_comment = column.get("comment", "")
# 判断是否为主键
is_pk = column_name in primary_keys
# 判断是否为唯一约束
is_unique = column_name in unique_columns
# 判断是否为自增列(基于数据库类型和列类型)
is_increment = column.get("autoincrement", False) in (True, "auto")
# 获取列长度(如果适用)
col_len = getattr(column["type"], "length", None)
column_length = str(col_len) if col_len is not None else ""
# 构造列信息字典
column_info = {
"column_name": column_name,
"column_comment": column_comment or "",
"column_type": column_type,
"column_length": column_length or "",
"column_default": str(column_default) if column_default is not None else "",
"sort": idx + 1, # 序号从1开始
"is_pk": bool(is_pk),
"is_increment": bool(is_increment),
"is_nullable": bool(is_nullable),
"is_unique": bool(is_unique),
}
# 构造列信息字典
column_info = {
"column_name": column_name,
"column_comment": column_comment or "",
"column_type": column_type,
"column_length": column_length or "",
"column_default": str(column_default) if column_default is not None else "",
"sort": idx + 1, # 序号从1开始
"is_pk": bool(is_pk),
"is_increment": bool(is_increment),
"is_nullable": bool(is_nullable),
"is_unique": bool(is_unique),
}
columns_list.append(column_info)
columns_list.append(column_info)
return columns_list
return columns_list
return await _run_inspector(_introspect)
async def get_gen_table_column_by_id(self, id: int, preload: list | None = None) -> GenTableColumnModel | None:
"""根据业务表字段ID获取业务表字段信息。
@@ -507,12 +523,7 @@ class GenTableColumnCRUD(CRUDBase[GenTableColumnModel, GenTableColumnSchema, Gen
raise ValueError("数据表名称不能为空")
try:
# 在线程池中执行同步 inspect 操作,避免阻塞事件循环
columns_info = await asyncio.to_thread(
GenTableColumnCRUD._sync_get_table_columns,
settings.DATABASE_TYPE,
table_name,
)
columns_info = await GenTableColumnCRUD._get_table_columns(settings.DATABASE_TYPE, table_name)
# 转换为GenTableColumnOutSchema对象列表
columns_list = [GenTableColumnOutSchema(**column_info) for column_info in columns_info]
@@ -14,7 +14,7 @@ from app.api.v1.module_system.log.model import LoginLogModel
from app.api.v1.module_system.log.schema import LoginLogCreateSchema
from app.api.v1.module_system.user.crud import UserCRUD
from app.api.v1.module_system.user.model import UserModel
from app.common.enums import RedisInitKeyConfig
from app.common.enums import RET, RedisInitKeyConfig
from app.config.setting import settings
from app.core.base_schema import AuthSchema, JWTOutSchema, JWTPayloadSchema
from app.core.database import async_db_session
@@ -165,6 +165,13 @@ class LoginService:
"""用户认证"""
ua_result = ua_parser.parse(request.headers.get("user-agent") or "")
request_ip = get_client_ip(request)
if redis is None:
# 会话存储不可用:登录即使成功也无法建立可用会话,明确 503
raise CustomException(
msg="Redis 服务不可用,无法建立登录会话",
code=RET.SERVICE_UNAVAILABLE.code,
status_code=503,
)
login_location = await IpLocalUtil.resolve_location_for_log(redis, request_ip)
# 暴力破解防护:先检查该用户名+IP 是否已被锁定(Redis 故障时跳过,避免锁死系统)
@@ -329,6 +336,13 @@ class LoginService:
@classmethod
async def create_token(cls, request: Request, redis: Redis, user: UserModel, login_type: str) -> JWTOutSchema:
"""创建访问令牌和刷新令牌"""
if redis is None:
# 会话信息仅存 Redis;降级模式下建出的 token 无法通过后续会话校验,直接 503
raise CustomException(
msg="Redis 服务不可用,无法建立登录会话",
code=RET.SERVICE_UNAVAILABLE.code,
status_code=503,
)
session_id = str(uuid.uuid4())
ua_result = ua_parser.parse(request.headers.get("user-agent") or "")
request_ip = get_client_ip(request)
@@ -406,6 +420,12 @@ class LoginService:
token_payload: JWTPayloadSchema = decode_access_token(token=refresh_token)
if not token_payload.is_refresh:
raise CustomException(msg="非法凭证,请传入刷新令牌")
if redis is None:
raise CustomException(
msg="Redis 服务不可用,无法刷新会话",
code=RET.SERVICE_UNAVAILABLE.code,
status_code=503,
)
session_id = token_payload.sub
session_info = await RedisCURD(redis).get(f"{RedisInitKeyConfig.USER_SESSION.key}:{session_id}")
@@ -379,17 +379,27 @@ class DictDataService:
logger.error(f"❌️ 字典初始化过程发生错误: {e}")
raise CustomException(msg="字典数据初始化失败") from e
@staticmethod
async def _load_dict_type_from_db(dict_type: str) -> list[dict]:
"""Redis 不可用时直接从数据库读取字典数据(序列化与 init_cache 一致)。"""
async with async_db_session() as session, session.begin():
init_auth = AuthSchema()
data_list = await DictDataCRUD(init_auth, session).get_list(search={"dict_type": dict_type})
return [DictDataOutSchema.model_validate(row).model_dump(mode="json") for row in data_list if row]
@staticmethod
async def get_init_cache(redis: Redis, dict_type: str) -> list[dict]:
"""从缓存获取字典数据列表信息(无 auth)。
参数:
- redis (Redis): Redis客户端
- redis (Redis): Redis客户端(降级模式下为 None,直接回源数据库)
- dict_type (str): 字典类型
返回:
- list[dict]: 字典数据列表
"""
if redis is None:
return await DictDataService._load_dict_type_from_db(dict_type)
def _parse(data: str | list | None) -> list[dict] | None:
"""尝试反序列化 Redis 返回的字典缓存数据"""