feat: PubMed搜索合规审计 Phase 5-7 全部修复
CI / backend (push) Canceled after 0s
CI / frontend (push) Canceled after 0s

P1-2: [SB][STAT][UID][DEP][MESH]字段支持
P1-4: [MH:noexp]解析+抑制树展开
P3: 前端date_to/P3-1、CJK高亮/P3-3、extractPlainText/P3-4、precision_mode移除/P3-6、loadMore sort/P3-5
P4-1: 词数上限50→100
P4-2: retracted参数清理
This commit is contained in:
34047007@qq.com
2026-07-27 09:03:19 +08:00
parent 62ca8fa6b8
commit 5f69277eba
7 changed files with 111 additions and 30 deletions
+2 -2
View File
@@ -89,8 +89,8 @@ class AdvancedSearchRequest(BaseModel):
@field_validator('query')
@classmethod
def check_query_complexity(cls, v: str) -> str:
if len(v.split()) > 50:
raise ValueError('查询词过多(最多 50 个词),请简化搜索条件')
if len(v.split()) > 100:
raise ValueError('查询词过多(最多 100 个词),请简化搜索条件')
return v
+24 -2
View File
@@ -23,7 +23,7 @@ from dataclasses import dataclass, field
from enum import Enum, auto
# ─── 查询复杂度限制 ───
MAX_TERMS = 50 # 与 API 层 check_query_complexity 的 50 词限制对齐
MAX_TERMS = 100 # P4-1: 放宽到 100 词(原 50 词
MAX_PAREN_DEPTH = 10 # 括号嵌套最深层数
@@ -42,6 +42,7 @@ _FIELD_TAG_MAP: dict[str, str] = {
"LAU": "author",
"TW": "all",
"OT": "all",
"MESH": "MH", # P1-2: [MESH] 是 [MH] 的别名
# 新增标量字段
"LA": "language",
"VI": "volume",
@@ -55,23 +56,30 @@ _SPECIAL_FIELDS = {
"MH", "MAJR", "PT", "DP", "PMID", "DOI",
# 日期字段
"EDAT", "CRDT", "MHDA", "LR", "DCOM",
# P1-2: 电子出版日期
"DEP",
# JSONB 字段
"GR", "SH", "RN", "NM", "SI", "PA",
# 待补充抽取的字段(标记为特殊以便未来实现)
"AUID", "COIS", "ED", "IR", "PS", "PUBN", "TT",
# P1-2: 新字段
"SB", "STAT", "UID",
}
# 支持日期范围语法的字段
_DATE_RANGE_FIELDS = {"DP", "EDAT", "CRDT", "MHDA", "LR", "DCOM"}
_DATE_RANGE_FIELDS = {"DP", "EDAT", "CRDT", "MHDA", "LR", "DCOM", "DEP"}
# 所有合法字段标签(PubMed 全量字段)
# P1-1: 移除了 BOOK/FILTER/ISBN(未实现,降级为 plain text 不如报错透明)
_ALL_FIELD_TAGS = {
"AB", "AD", "AU", "CN", "FAU", "AUID", "LAU", "COIS",
"DCOM", "CRDT", "EDAT", "MHDA", "LR", "DP", "DOI",
"DEP", # P1-2: Date of Electronic Publication
"RN", "ED", "GR", "IR", "IP",
"TA", "JT", "LA", "LID", "MAJR", "SH", "MH", "MH:NOEXP", "OT", "PG",
"PA", "PT", "PMID", "PUBN", "SI", "PS", "NM", "TW",
"SB", "STAT", "UID", # P1-2: Subset, Status, UID
"MESH", # P1-2: [MH] 别名
"TI", "TIAB", "TT", "VI",
}
@@ -187,6 +195,8 @@ class ParsedPubmedQuery:
lr_to: str | None = None
dcom_from: str | None = None # [DCOM]
dcom_to: str | None = None
dep_from: str | None = None # P1-2: [DEP]
dep_to: str | None = None
# 特殊字段(存储 Term 以保留 is_not 标志)
mesh_terms: list[Term] = field(default_factory=list) # [MH] (was list[str])
majr_terms: list[Term] = field(default_factory=list) # [MAJR] (was list[str])
@@ -207,6 +217,9 @@ class ParsedPubmedQuery:
auid_terms: list[Term] = field(default_factory=list) # [AUID]
cois_terms: list[Term] = field(default_factory=list) # [COIS]
tt_terms: list[Term] = field(default_factory=list) # [TT]
sb_terms: list[Term] = field(default_factory=list) # P1-2: [SB] Subset
stat_terms: list[Term] = field(default_factory=list) # P1-2: [STAT] Status
uid_terms: list[Term] = field(default_factory=list) # P1-2: [UID] PMID/DOI 统一
plain_terms: list[Term] = field(default_factory=list) # no field tag
boolean_operator: str = "and" # "and" | "or" | "mixed"
has_not: bool = False # contains NOT
@@ -349,6 +362,8 @@ class PubmedQueryParser:
pass
elif term.field == "__RANGE_DCOM__":
pass
elif term.field == "__RANGE_DEP__":
pass
elif term.field == "ED":
result.ed_terms.append(term)
elif term.field == "IR":
@@ -363,6 +378,12 @@ class PubmedQueryParser:
result.cois_terms.append(term)
elif term.field == "TT":
result.tt_terms.append(term)
elif term.field == "SB":
result.sb_terms.append(term)
elif term.field == "STAT":
result.stat_terms.append(term)
elif term.field == "UID":
result.uid_terms.append(term)
else:
result.plain_terms.append(term)
@@ -494,6 +515,7 @@ class PubmedQueryParser:
"MHDA": ("mhda_from", "mhda_to", None, None, "__RANGE_MHDA__"),
"LR": ("lr_from", "lr_to", None, None, "__RANGE_LR__"),
"DCOM": ("dcom_from", "dcom_to", None, None, "__RANGE_DCOM__"),
"DEP": ("dep_from", "dep_to", None, None, "__RANGE_DEP__"), # P1-2
}
date_attr, date_attr_to, yr_from_attr, yr_to_attr, marker_field = attr_map[field]
# 反向范围自动交换(如 2026:2024[DP] → 2024:2026[DP]
+68 -15
View File
@@ -97,7 +97,7 @@ class AdvancedSearchEngine:
journal_tiers: list[str] | None = None,
pub_types: list[str] | None = None,
tag_ids: list[str] | None = None,
retracted: str = "", # "yes", "no", "only"
retracted: str = "", # "no", "only"
negative_result: str = "", # "yes", "no", "only"
is_oa: bool | None = None, # True = 仅开放获取
language: str | None = None, # 语言代码(en/zh/fr 等,向后兼容)
@@ -820,6 +820,42 @@ class AdvancedSearchEngine:
neg_conds = [GlobalLiterature.vernacular_title.ilike(f"%{_escape_ilike(t.text)}%") for t in neg]
term_conditions.append(not_(or_(*neg_conds)))
# P1-2: [SB] Subset → nlm_subsets(需 journal JOIN 子查询)
if pp.sb_terms:
pos = [t for t in pp.sb_terms if not t.is_not]
neg = [t for t in pp.sb_terms if t.is_not]
for subl, is_neg in [(pos, False), (neg, True)]:
for t in subl:
subq = select(GlobalJournal.issn).where(
GlobalJournal.nlm_subsets.overlap([t.text.upper()])
)
cond = GlobalLiterature.journal_issn.in_(subq)
term_conditions.append(not_(cond) if is_neg else cond)
# P1-2: [STAT] Status → citation_status
if pp.stat_terms:
pos = [t for t in pp.stat_terms if not t.is_not]
neg = [t for t in pp.stat_terms if t.is_not]
if pos:
term_conditions.append(or_(*[
GlobalLiterature.citation_status == t.text.lower()
for t in pos
]))
if neg:
neg_conds = [GlobalLiterature.citation_status == t.text.lower() for t in neg]
term_conditions.append(not_(or_(*neg_conds)))
# P1-2: [UID] → PMID 优先,兜底 DOI
if pp.uid_terms:
for t in pp.uid_terms:
cond = None
try:
cond = GlobalLiterature.pmid == int(t.text)
except ValueError:
cond = GlobalLiterature.doi.ilike(f"%{_escape_ilike(t.text)}%")
if cond is not None:
term_conditions.append(not_(cond) if t.is_not else cond)
# 处理括号分组的词(保留 OR/AND 嵌套结构,P2-2
if pp.groups:
for idx, group in enumerate(pp.groups):
@@ -881,13 +917,14 @@ class AdvancedSearchEngine:
cond = and_(*dp_conds) if len(dp_conds) > 1 else dp_conds[0]
conditions.append(not_(cond) if dp_negated else cond)
# 6b. [EDAT] [CRDT] [MHDA] [LR] [DCOM] → 日期字段范围
# 6b. [EDAT] [CRDT] [MHDA] [LR] [DCOM] [DEP] → 日期字段范围
DATE_FIELD_COLS = {
"edat": (GlobalLiterature.entrez_date, "EDAT"),
"crdt": (GlobalLiterature.create_date, "CRDT"),
"mhda": (GlobalLiterature.meshed_date, "MHDA"),
"lr": (GlobalLiterature.pubmed_revised, "LR"),
"dcom": (GlobalLiterature.date_completed, "DCOM"),
"dep": (GlobalLiterature.pub_date, "DEP"), # P1-2: [DEP] → pub_date
}
for prefix, (col, field_tag) in DATE_FIELD_COLS.items():
_from = getattr(pp, f"{prefix}_from", None)
@@ -942,7 +979,7 @@ class AdvancedSearchEngine:
# 特殊字段 — 与 _pubmed_conditions 中 top-level dispatch 一致
if field == "MH":
return await AdvancedSearchEngine._expand_mesh_tag_ids(db, [term.text], major_only=False)
return await AdvancedSearchEngine._expand_mesh_tag_ids(db, [term.text], major_only=False, noexp=term._noexp)
if field == "MAJR":
return await AdvancedSearchEngine._expand_mesh_tag_ids(db, [term.text], major_only=True)
if field == "PT":
@@ -980,6 +1017,18 @@ class AdvancedSearchEngine:
return GlobalLiterature.cois_statement.ilike(f"%{_escape_ilike(term.text)}%")
if field == "TT":
return GlobalLiterature.vernacular_title.ilike(f"%{_escape_ilike(term.text)}%")
if field == "SB":
subq = select(GlobalJournal.issn).where(
GlobalJournal.nlm_subsets.overlap([term.text.upper()])
)
return GlobalLiterature.journal_issn.in_(subq)
if field == "STAT":
return GlobalLiterature.citation_status == term.text.lower()
if field == "UID":
try:
return GlobalLiterature.pmid == int(term.text)
except ValueError:
return GlobalLiterature.doi.ilike(f"%{_escape_ilike(term.text)}%")
# 回退
return AdvancedSearchEngine._field_condition("all", term.text, term.exact)
@@ -1051,6 +1100,7 @@ class AdvancedSearchEngine:
db: AsyncSession,
mesh_names: list[str],
major_only: bool = False,
noexp: bool = False, # P1-4: [MH:noexp] 抑制树展开
) -> object | None:
"""用 GlobalTagTreeNumber 展开 [MH]/[MAJR]
@@ -1058,6 +1108,8 @@ class AdvancedSearchEngine:
2. 用 tree_number 前缀展开子节点(C04.588 → 所有子树号)
3. [MAJR] 额外 AND is_major=True
返回 SQLAlchemy condition 或 None(无匹配时)。
noexp=True 时跳过第 2 步(树展开),只搜精确词。
"""
import uuid as _uuid
mesh_tag_ids: set[_uuid.UUID] = set()
@@ -1091,19 +1143,20 @@ class AdvancedSearchEngine:
if not mesh_tag_ids:
return None
# tree_number 前缀展开:取匹配 tag 的所有 tree_number,查子节点
tns = (await db.execute(
select(GlobalTagTreeNumber.tree_number).where(
GlobalTagTreeNumber.tag_id.in_(list(mesh_tag_ids))
).distinct()
)).scalars().all()
if tns:
child_conds = [GlobalTagTreeNumber.tree_number.like(f"{tn}%") for tn in tns]
children = (await db.execute(
select(GlobalTagTreeNumber.tag_id).where(or_(*child_conds))
# tree_number 前缀展开:取匹配 tag 的所有 tree_number,查子节点[MH:noexp] 时跳过)
if not noexp:
tns = (await db.execute(
select(GlobalTagTreeNumber.tree_number).where(
GlobalTagTreeNumber.tag_id.in_(list(mesh_tag_ids))
).distinct()
)).scalars().all()
mesh_tag_ids.update(children)
if tns:
child_conds = [GlobalTagTreeNumber.tree_number.like(f"{tn}%") for tn in tns]
children = (await db.execute(
select(GlobalTagTreeNumber.tag_id).where(or_(*child_conds))
)).scalars().all()
mesh_tag_ids.update(children)
uids = list(mesh_tag_ids)
if major_only:
@@ -301,9 +301,10 @@ class TestAllFrontendQueryFormats:
def test_field_map_matches_search_engine(self):
"""Verify field tag map covers all search engine field types."""
from app.services.pubmed_query_parser import _FIELD_TAG_MAP
from app.services.pubmed_query_parser import _FIELD_TAG_MAP, _SPECIAL_FIELDS
supported_fields = {"title", "abstract", "all", "author", "journal", "affiliation",
"language", "volume", "issue", "pages", "lid"}
supported_fields.update(_SPECIAL_FIELDS) # 特殊字段由 _single_term_condition 处理
for fld in set(_FIELD_TAG_MAP.values()):
assert fld in supported_fields, f"Mapped field '{fld}' not supported"
@@ -111,8 +111,11 @@ const nctId = computed(() => {
// 从查询中提取纯文本词(去掉 PubMed 字段标签如 [TI]、[AB] 等)
function extractPlainText(q: string): string {
return q
.replace(/\[[\w-]+\]/g, '')
.replace(/"?\b(AND|OR|NOT)\b"?/gi, '')
.replace(/\[[\w-]+\]/g, '') // [field tags]
.replace(/"?\b(AND|OR|NOT)\b"?/gi, '')// boolean operators
.replace(/[()]/g, '') // P3-4: 去掉括号
.replace(/#\d+/g, '') // P3-4: 去掉 #N 引用标记
.replace(/\*/g, '') // P3-4: 去掉通配符
.replace(/"/g, '')
.replace(/\s+/g, ' ')
.trim()
@@ -127,8 +130,9 @@ const highlightedTitle = computed(() => {
const terms = plain.split(/\s+/).filter(t => t.length > 0)
if (terms.length === 0) return ''
const escaped = terms.map(t => t.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
// 单字符词添加 \b 词边界避免过匹配("A" 不匹配每个字母 "a"
const pattern = escaped.map(e => e.length === 1 ? `\\b${e}\\b` : e).join('|')
// P3-3: 单字符词添加 \b 词边界,但 CJK 字符跳过(\b 对 CJK 无效
const isCJK = (c: string) => /[-鿿㐀-䶿豈-﫿]/.test(c)
const pattern = escaped.map(e => e.length === 1 && !isCJK(e) ? `\\b${e}\\b` : e).join('|')
const parts = title.split(new RegExp(`(${pattern})`, 'gi'))
return parts.map((part: string) =>
terms.some(t => part.toLowerCase() === t.toLowerCase())
+4 -3
View File
@@ -277,7 +277,7 @@ const { page, total, goToPage } = usePagination({
query: query.value, page: p, page_size: pageSize.value, sort: sort.value,
}
if (field.value !== 'all') body.field = field.value
if (precisionMode.value !== 'majr') body.precision_mode = precisionMode.value
// P3-6: precision_mode 不再发送(后端已忽略)
// ── Keyset 游标分页(sort=date 时跳过 COUNT,纯翻页模式) ──
const useKeyset = sort.value === 'date'
if (useKeyset) {
@@ -371,10 +371,11 @@ function restoreFromQuery() {
datePreset.value = String(route.query.date_preset)
} else if (route.query.date_from) {
datePreset.value = null
// 尝试解析 date_from/date_to 到 yearFromStr/yearToStr
const df = String(route.query.date_from)
if (df.length >= 4) yearFromStr.value = df.slice(0, 4)
} else if (route.query.date_to) {
}
if (route.query.date_to) {
datePreset.value = null
const dt = String(route.query.date_to)
if (dt.length >= 4) yearToStr.value = dt.slice(0, 4)
}
+3 -3
View File
@@ -214,7 +214,7 @@ function doLocalSearch() {
if (searchParams.value.date_from) query.date_from = searchParams.value.date_from
if (searchParams.value.date_to) query.date_to = searchParams.value.date_to
if (searchParams.value.sort && searchParams.value.sort !== 'date') query.sort = searchParams.value.sort
if (searchParams.value.precision_mode && searchParams.value.precision_mode !== 'majr') query.precision = searchParams.value.precision_mode
// P3-6: precision_mode 不再发送
if (searchParams.value.retracted) query.retracted = searchParams.value.retracted
if (searchParams.value.negative_result) query.negative = searchParams.value.negative_result
if (selectedTagIds.value.length > 0) query.tag = selectedTagIds.value.join(',')
@@ -319,7 +319,7 @@ async function loadMore() {
if (loadingMore.value || !hasMore.value) return
loadingMore.value = true
try {
const body: Record<string, any> = { page_size: searchParams.value.page_size, sort: 'date' }
const body: Record<string, any> = { page_size: searchParams.value.page_size, sort: searchParams.value.sort || 'date' }
if (searchParams.value.query) body.query = searchParams.value.query
if (searchParams.value.field !== 'all') body.field = searchParams.value.field
if (searchParams.value.tag_ids.length) body.tag_ids = searchParams.value.tag_ids
@@ -327,7 +327,7 @@ async function loadMore() {
if (searchParams.value.date_to) body.date_to = searchParams.value.date_to
if (searchParams.value.retracted) body.retracted = searchParams.value.retracted
if (searchParams.value.negative_result) body.negative_result = searchParams.value.negative_result
if (searchParams.value.precision_mode) body.precision_mode = searchParams.value.precision_mode
// P3-6: precision_mode 不再发送
if (cursorDate.value && cursorId.value) {
body.cursor_date = cursorDate.value
body.cursor_id = cursorId.value