checkpoint: 架构治理四件套——统一 async / 测试真 PG / Redis 降级可选 / DPB 扶正
- A1 业务层统一 async：gencode 表内省改 run_sync 异步（不再阻塞事件循环）、 check_db 改 async_engine、dict_util 启动预热异步载入；psycopg 降为 APScheduler SQLAlchemyJobStore 专用同步孤岛（注释标注） - A2 测试真 PG 化：conftest 弃 SQLite + mock Redis，改连真实 PG16(dpb_test)+ Redis（number_gen advisory lock 路径真被测）；新增 docker/test-compose.yaml 测试依赖栈(postgres:16:5433 + redis:7:6380)；run_ci 前置探测 + 容器兜底， pytest -m pg 10 套正式 tc 全过（真实 PG16 + Redis） - B1 Redis 启动不强依赖：redis_connect 失败降级启动，缓存类回源 DB、存储类 (会话/AI 配置/调度)经 require_redis 守卫返回 503；真机 3 阶段降级测试 PASS=11 FAIL=0（独立 Redis 6390 + 后端 8091，不动共享 dev 服务） - C 扶正 DPB：后端横幅/日志(dpb.log)/README/pyproject/ai_factory agent 名、 前端 package.json/署名注释/链接文案/deploy.sh/docker 注释全部去 fastapiadmin； grep backend/app + frontend/web/src 零残留 - 验证：全量回归 pass=72 fail=0
This commit is contained in:
34047007@qq.com
+85
-132
@@ -1,187 +1,138 @@
|
||||
"""conftest — 模块化 API 接口测试共享 fixture。
|
||||
"""conftest — 模块化 API 接口测试共享 fixture（真实 PostgreSQL + Redis）。
|
||||
|
||||
A2 架构治理：冒烟层不再用 SQLite + mock Redis，改为连真实 PostgreSQL16（专用
|
||||
测试库 dpb_test，number_gen 的 pg_advisory_xact_lock 路径真被执行，不再静默降级）
|
||||
+ 真实 Redis，与统计引擎 tc 套件同一套基础设施（可被 TC_CI_* 环境变量覆盖，供 CI
|
||||
容器指定宿主）。
|
||||
|
||||
提供:
|
||||
- test_client: FastAPI TestClient 实例 (session 级复用)
|
||||
- assert_route: 验证接口路由存在 (status_code != 404)
|
||||
|
||||
注意: 本 conftest 强制 DATABASE_TYPE=sqlite + mock Redis，仅作用于 backend/tests/ 目录。
|
||||
backend/scripts/ 下的 test_bre_analysis_*_tc.py 是独立进程的真实 PG16+Redis 套件
|
||||
（由 tests/test_stat_analysis_tc.py 以 subprocess 调用），此处用 collect_ignore_glob
|
||||
排除，避免 pytest 收集时 import 它们污染 sqlite 环境。
|
||||
排除，避免 pytest 收集时 import 它们。
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
collect_ignore_glob = ["../scripts/test_*.py"]
|
||||
import asyncio
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import AsyncGenerator, Generator
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
# ============================================================
|
||||
# 测试环境变量 —— 必须在 import app 之前设置（引擎在 app.core.database
|
||||
# 模块级按 settings 构建，settings 读 ENVIRONMENT 对应的 .env.dev + os.environ）
|
||||
# ============================================================
|
||||
|
||||
os.environ.setdefault("ENVIRONMENT", "dev")
|
||||
os.environ["DATABASE_TYPE"] = "postgres"
|
||||
os.environ["DATABASE_NAME"] = os.environ.get("TC_CI_DB_NAME", "dpb_test")
|
||||
os.environ["DATABASE_HOST"] = os.environ.get("TC_CI_DB_HOST", "localhost")
|
||||
os.environ["DATABASE_PORT"] = os.environ.get("TC_CI_DB_PORT", "5432")
|
||||
os.environ["DATABASE_USER"] = os.environ.get("TC_CI_DB_USER", "dpb")
|
||||
os.environ["DATABASE_PASSWORD"] = os.environ.get("TC_CI_DB_PASSWORD", "dpb")
|
||||
for ci_key, app_key in {
|
||||
"TC_CI_REDIS_HOST": "REDIS_HOST",
|
||||
"TC_CI_REDIS_PORT": "REDIS_PORT",
|
||||
"TC_CI_REDIS_DB": "REDIS_DB_NAME",
|
||||
}.items():
|
||||
if os.environ.get(ci_key):
|
||||
os.environ[app_key] = os.environ[ci_key]
|
||||
os.environ["CAPTCHA_ENABLE"] = "False" # 测试环境关闭验证码
|
||||
|
||||
|
||||
def _ensure_test_db() -> None:
|
||||
"""确保测试库存在（连接维护库 postgres 按需 CREATE DATABASE，已存在则跳过）。
|
||||
|
||||
测试库默认 dpb_test，与开发库 dpb 隔离；TC_CI_DB_NAME 可指向容器库。
|
||||
"""
|
||||
dbname = os.environ["DATABASE_NAME"]
|
||||
if dbname in {"postgres", "template0", "template1"}:
|
||||
return
|
||||
import asyncpg
|
||||
|
||||
host = os.environ["DATABASE_HOST"]
|
||||
port = int(os.environ["DATABASE_PORT"])
|
||||
user = os.environ["DATABASE_USER"]
|
||||
password = os.environ.get("DATABASE_PASSWORD") or None
|
||||
|
||||
async def _go() -> None:
|
||||
admin = await asyncpg.connect(
|
||||
host=host, port=port, user=user, password=password, database="postgres", timeout=5
|
||||
)
|
||||
try:
|
||||
exists = await admin.fetchval("SELECT 1 FROM pg_database WHERE datname = $1", dbname)
|
||||
if not exists:
|
||||
await admin.execute(f'CREATE DATABASE "{dbname}"')
|
||||
finally:
|
||||
await admin.close()
|
||||
|
||||
asyncio.run(_go())
|
||||
|
||||
|
||||
_ensure_test_db()
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
# ============================================================
|
||||
# 测试环境变量
|
||||
# ============================================================
|
||||
|
||||
_TEST_DB_PATH = tempfile.NamedTemporaryFile(suffix=".db", delete=False).name
|
||||
|
||||
os.environ["DATABASE_TYPE"] = "sqlite"
|
||||
os.environ["DATABASE_NAME"] = _TEST_DB_PATH
|
||||
os.environ["POOL_SIZE"] = "1"
|
||||
os.environ["MAX_OVERFLOW"] = "1"
|
||||
|
||||
from app.config.setting import settings
|
||||
|
||||
settings.DATABASE_TYPE = "sqlite"
|
||||
settings.DATABASE_NAME = _TEST_DB_PATH
|
||||
settings.POOL_SIZE = 1
|
||||
settings.MAX_OVERFLOW = 1
|
||||
settings.CAPTCHA_ENABLE = False # 测试环境关闭验证码
|
||||
|
||||
# ============================================================
|
||||
# Mock Redis — dict 存储，支持 get/set/delete/exists/keys/ttl/expire
|
||||
# 登录成功后写入的 session 数据可在后续请求中正确读取
|
||||
# 跨事件循环安全 —— TestClient 的 lifespan 跑在 portal 线程 loop，而 pytest-asyncio
|
||||
# 的 async 测试跑在函数级新 loop；asyncpg 连接 loop 绑定，复用池化连接会
|
||||
# RuntimeError。改用 NullPool：每个 session 在当前 loop 新开连接、同 loop 关闭，
|
||||
# 永不跨 loop 复用（冒烟层 session 级仅一处 async 直连用例，代价可忽略）。
|
||||
# ============================================================
|
||||
|
||||
_mock_redis_store: dict[bytes, bytes] = {}
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
import app.core.database as _database
|
||||
|
||||
def _redis_get(name: bytes) -> bytes | None:
|
||||
return _mock_redis_store.get(name)
|
||||
_test_engine = create_async_engine(settings.ASYNC_DB_URI, poolclass=NullPool)
|
||||
_database.async_engine = _test_engine
|
||||
_database.async_db_session = async_sessionmaker(
|
||||
bind=_test_engine, class_=AsyncSession, expire_on_commit=settings.EXPIRE_ON_COMMIT
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# 调度器 mock —— 冒烟层不启动后台调度线程（真实调度由 tc 套件经真实 lifespan 覆盖）
|
||||
# ============================================================
|
||||
|
||||
async def _redis_set(name: bytes, value: bytes, ex: int | None = None, nx: bool = False) -> bool | None:
|
||||
if nx and name in _mock_redis_store:
|
||||
return None
|
||||
_mock_redis_store[name] = value
|
||||
return True
|
||||
|
||||
|
||||
async def _redis_delete(*names: bytes) -> int:
|
||||
count = 0
|
||||
for n in names:
|
||||
if _mock_redis_store.pop(n, None) is not None:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def _redis_keys(pattern: bytes | None = None) -> list[bytes]:
|
||||
if pattern == b"*" or pattern is None:
|
||||
return list(_mock_redis_store.keys())
|
||||
return [k for k in _mock_redis_store if pattern == b"*" or k.startswith(pattern.replace(b"*", b""))]
|
||||
|
||||
|
||||
def _redis_exists(*names: bytes) -> int:
|
||||
return sum(1 for n in names if n in _mock_redis_store)
|
||||
|
||||
|
||||
def _redis_ttl(name: bytes) -> int:
|
||||
return 3600 if name in _mock_redis_store else -2
|
||||
|
||||
|
||||
async def _redis_expire(name: bytes, time: int) -> bool:
|
||||
return name in _mock_redis_store
|
||||
|
||||
|
||||
async def _redis_flushall(asynchronous: bool = False) -> bool:
|
||||
_mock_redis_store.clear()
|
||||
return True
|
||||
|
||||
|
||||
async def _redis_flushdb(asynchronous: bool = False) -> bool:
|
||||
_mock_redis_store.clear()
|
||||
return True
|
||||
|
||||
|
||||
async def _redis_close() -> None:
|
||||
pass
|
||||
|
||||
|
||||
async def _redis_aclose() -> None:
|
||||
pass
|
||||
|
||||
|
||||
async def _redis_hmget(name: bytes, keys: list[bytes]) -> list[bytes | None]:
|
||||
return [_mock_redis_store.get(name + b":" + k) for k in keys]
|
||||
|
||||
|
||||
async def _redis_hset(name: bytes, key: bytes, value: bytes) -> int:
|
||||
_mock_redis_store[name + b":" + key] = value
|
||||
return 1
|
||||
|
||||
|
||||
async def _redis_hgetall(name: bytes) -> dict[bytes, bytes]:
|
||||
prefix = name + b":"
|
||||
return {k[len(prefix) :]: v for k, v in _mock_redis_store.items() if k.startswith(prefix)}
|
||||
|
||||
|
||||
async def _redis_hdel(name: bytes, *keys: bytes) -> int:
|
||||
count = 0
|
||||
for k in keys:
|
||||
if _mock_redis_store.pop(name + b":" + k, None) is not None:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def _redis_info(section: str | None = None) -> dict:
|
||||
return {}
|
||||
|
||||
|
||||
def _redis_dbsize() -> int:
|
||||
return len(_mock_redis_store)
|
||||
|
||||
|
||||
_mock_redis = AsyncMock()
|
||||
_mock_redis.ping = AsyncMock(return_value=True)
|
||||
_mock_redis.get = AsyncMock(side_effect=_redis_get)
|
||||
_mock_redis.set = AsyncMock(side_effect=_redis_set)
|
||||
_mock_redis.delete = AsyncMock(side_effect=_redis_delete)
|
||||
_mock_redis.keys = AsyncMock(side_effect=_redis_keys)
|
||||
_mock_redis.exists = AsyncMock(side_effect=_redis_exists)
|
||||
_mock_redis.ttl = AsyncMock(side_effect=_redis_ttl)
|
||||
_mock_redis.expire = AsyncMock(side_effect=_redis_expire)
|
||||
_mock_redis.flushall = AsyncMock(side_effect=_redis_flushall)
|
||||
_mock_redis.flushdb = AsyncMock(side_effect=_redis_flushdb)
|
||||
_mock_redis.close = AsyncMock(side_effect=_redis_close)
|
||||
_mock_redis.aclose = AsyncMock(side_effect=_redis_aclose)
|
||||
_mock_redis.hmget = AsyncMock(side_effect=_redis_hmget)
|
||||
_mock_redis.hset = AsyncMock(side_effect=_redis_hset)
|
||||
_mock_redis.hgetall = AsyncMock(side_effect=_redis_hgetall)
|
||||
_mock_redis.hdel = AsyncMock(side_effect=_redis_hdel)
|
||||
_mock_redis.info = AsyncMock(side_effect=_redis_info)
|
||||
_mock_redis.dbsize = AsyncMock(side_effect=_redis_dbsize)
|
||||
|
||||
patch("redis.asyncio.Redis.from_url", return_value=_mock_redis).start()
|
||||
# slowapi 限流器由中间件处理，测试中无需 mock
|
||||
patch("app.core.ap_scheduler.SchedulerUtil.init_scheduler", new=AsyncMock()).start()
|
||||
patch("app.core.ap_scheduler.SchedulerUtil.shutdown", new=AsyncMock()).start()
|
||||
|
||||
# slowapi 通过中间件限流，测试无需 mock
|
||||
|
||||
# ============================================================
|
||||
# 精简 lifespan — 仅做数据库初始化
|
||||
# 精简 lifespan — 真实 PG + Redis，仅跳过调度器
|
||||
# ============================================================
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _test_lifespan(app) -> AsyncGenerator[Any, None]:
|
||||
from app.api.v1.module_system.dict.service import DictDataService
|
||||
from app.api.v1.module_system.params.service import ParamsService
|
||||
from app.scripts.initialize import InitializeData
|
||||
from app.core.database import async_db_session, redis_connect
|
||||
|
||||
await InitializeData().init_db()
|
||||
app.state.redis = _mock_redis
|
||||
await redis_connect(app, status=True)
|
||||
await ParamsService.init_cache(redis=app.state.redis)
|
||||
await DictDataService.init_cache(redis=app.state.redis)
|
||||
|
||||
# 将 admin 密码重置为已知密码 "admin123"
|
||||
from sqlalchemy import update
|
||||
|
||||
from app.api.v1.module_system.user.model import UserModel
|
||||
from app.core.database import async_db_session
|
||||
from app.utils.password_util import PwdUtil
|
||||
|
||||
async with async_db_session() as db:
|
||||
@@ -190,6 +141,8 @@ async def _test_lifespan(app) -> AsyncGenerator[Any, None]:
|
||||
|
||||
yield
|
||||
|
||||
await redis_connect(app, status=False)
|
||||
|
||||
|
||||
from main import create_app
|
||||
|
||||
|
||||
Reference in New Issue
Block a user