fix: ATM/Mesh 标签子查询 IN→关联 EXISTS — 生产 cancer 搜索 15s 超时修复
CI / backend (push) Waiting to run
CI / frontend (push) Waiting to run

根因:flat-text ATM 和 [MH] 展开返回 id IN (SELECT literature_id FROM
global_literature_tags WHERE tag_id IN (...))。当该子查询与 tsvector/ILIKE
文本条件 OR 组合进 Filter 时,planner 因命中行数过大(生产 'cancer' 展开 51
tag → 47.7 万 literature_id)拒绝哈希为 initplan,改为 Materialize 反复重扫
Bitmap Heap Scan 54-75 次,主查询 15.3s 超时(本地仅 6.4 万行哈希一次 156ms)。

修复:改为关联 EXISTS(literature_id = global_literature.id AND tag_id IN),
每外行一次 PK(literature_id, tag_id) 索引探测。生产 EXPLAIN 验证:
IN+OR 15260ms → EXISTS+OR 43.7ms(350×)。三处同模式一并修复:
- query_expansion.expand_atm(flat-text ATM)
- search_engine._expand_mesh_tag_ids([MH]/[MAJR])
- literature.search_literature 中文搜索标签注入
本地 IN/EXISTS 结果集逐行等价验证;全量 1007 tests passed。
This commit is contained in:
34047007@qq.com
2026-08-11 08:40:15 +08:00
parent dba5aebd87
commit d4853f7f3f
3 changed files with 21 additions and 21 deletions
+5 -4
View File
@@ -7,7 +7,7 @@ from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, Query from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi.responses import PlainTextResponse from fastapi.responses import PlainTextResponse
from pydantic import BaseModel from pydantic import BaseModel
from sqlalchemy import case, func, literal, or_, select, text from sqlalchemy import case, exists, func, literal, or_, select, text
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.core.permissions import get_current_user from app.core.permissions import get_current_user
@@ -289,10 +289,11 @@ async def search_literature(
).limit(100) ).limit(100)
)).scalars().all() )).scalars().all()
if _tag_matches: if _tag_matches:
_tag_lit_subq = select(GlobalLiteratureTag.literature_id).where( _tag_exist = exists().where(
GlobalLiteratureTag.tag_id.in_(list(_tag_matches)) GlobalLiteratureTag.literature_id == GlobalLiterature.id,
GlobalLiteratureTag.tag_id.in_(list(_tag_matches)),
) )
search_cond = or_(search_cond, GlobalLiterature.id.in_(_tag_lit_subq)) search_cond = or_(search_cond, _tag_exist)
# COUNT:使用 LIMIT 10001 截断,避免大结果集的精确计数 # COUNT:使用 LIMIT 10001 截断,避免大结果集的精确计数
# 结果集 ≤10000 时返回精确值,否则返回 10001+ # 结果集 ≤10000 时返回精确值,否则返回 10001+
MAX_EXACT = 10000 MAX_EXACT = 10000
+6 -8
View File
@@ -20,7 +20,7 @@ import logging
import re import re
from uuid import UUID from uuid import UUID
from sqlalchemy import or_ as _or_, select from sqlalchemy import exists, or_ as _or_, select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.models.literature import GlobalTag, GlobalTagTreeNumber, GlobalLiteratureTag, GlobalLiterature from app.models.literature import GlobalTag, GlobalTagTreeNumber, GlobalLiteratureTag, GlobalLiterature
@@ -53,13 +53,11 @@ async def expand_atm(db: AsyncSession, query: str):
if not expanded_ids: if not expanded_ids:
return None return None
# 不用 .distinct()IN (subquery) 语义本身就按值去重,显式 DISTINCT 只会让 PG # 不用 .distinct()EXISTS 关联子查询按 literature_id 探测 PK 索引(每外行一次
# 先物化+排序海量 literature_id'cancer' 这类词展开出 150+ 个 tag,命中的 # 索引查找),不走 ix_glt_tag 全量物化。
# literature_id 可达数百万),再对 id IN (百万集合) 扫描。去掉后可直接走 return exists().where(
# ix_glt_tag 索引半连接,year_counts 全量扫描和主查询都受益。 GlobalLiteratureTag.literature_id == GlobalLiterature.id,
return GlobalLiterature.id.in_( GlobalLiteratureTag.tag_id.in_(expanded_ids),
select(GlobalLiteratureTag.literature_id)
.where(GlobalLiteratureTag.tag_id.in_(expanded_ids))
) )
+10 -9
View File
@@ -2190,19 +2190,20 @@ class AdvancedSearchEngine:
return None return None
uids = list(mesh_tag_ids) uids = list(mesh_tag_ids)
# 不用 func.distinctIN (subquery) 语义本身就按值去重,显式 DISTINCT 只让 # 不用 .distinct()EXISTS 关联子查询按 literature_id 探测 PK 索引(每外行一次
# PG 多一次物化排序;去掉后可直接走 ix_glt_tag 索引半连接(year_counts 等 # 索引查找),不走 ix_glt_tag 全量物化。IN (subquery) 在 OR 过滤器里会被 planner
# 全量扫描场景更敏感)。 # 反复重扫('cancer' 展开命中的 literature_id 达数十万,Materialize 循环数十次
# 直接 15s 超时),EXISTS 关联每次外行只做一次 PK 探测。
if major_only: if major_only:
subq = select(GlobalLiteratureTag.literature_id).where( return exists().where(
GlobalLiteratureTag.literature_id == GlobalLiterature.id,
GlobalLiteratureTag.tag_id.in_(uids), GlobalLiteratureTag.tag_id.in_(uids),
GlobalLiteratureTag.is_major == True, GlobalLiteratureTag.is_major == True,
) )
else: return exists().where(
subq = select(GlobalLiteratureTag.literature_id).where( GlobalLiteratureTag.literature_id == GlobalLiterature.id,
GlobalLiteratureTag.tag_id.in_(uids), GlobalLiteratureTag.tag_id.in_(uids),
) )
return GlobalLiterature.id.in_(subq)
@staticmethod @staticmethod
def _best_match_order(tsq): def _best_match_order(tsq):