fix: 第29轮搜索审计修复 — 缺失导入/中文精确短语ILIKE/子组日期标记/开区间范围/NULL安全等8项
This commit is contained in:
@@ -932,6 +932,19 @@ class PubmedQueryParser:
|
|||||||
t1 = self.peek_n(1)
|
t1 = self.peek_n(1)
|
||||||
t2 = self.peek_n(2)
|
t2 = self.peek_n(2)
|
||||||
|
|
||||||
|
# R29: open-ended start range :2024[DP] → COLON NUMBER/DATE FIELD
|
||||||
|
if t0.type == TokenType.COLON and t1 is not None and t1.type in (TokenType.NUMBER, TokenType.DATE):
|
||||||
|
self.advance() # consume COLON
|
||||||
|
e_val = self.advance().value # end value
|
||||||
|
return self._handle_date_range_edge(result, negated, start_val=None, end_val=e_val)
|
||||||
|
|
||||||
|
# R29: open-ended end range 2024:[DP] → NUMBER/DATE COLON FIELD
|
||||||
|
if (t0.type in (TokenType.NUMBER, TokenType.DATE) and t1 is not None and t1.type == TokenType.COLON
|
||||||
|
and t2 is not None and t2.type == TokenType.FIELD):
|
||||||
|
s_val = self.advance().value # start value
|
||||||
|
self.advance() # consume COLON
|
||||||
|
return self._handle_date_range_edge(result, negated, start_val=s_val, end_val=None)
|
||||||
|
|
||||||
if (t1 is not None and t1.type == TokenType.COLON
|
if (t1 is not None and t1.type == TokenType.COLON
|
||||||
and t2 is not None
|
and t2 is not None
|
||||||
and t0.type in (TokenType.NUMBER, TokenType.DATE, TokenType.WORD)
|
and t0.type in (TokenType.NUMBER, TokenType.DATE, TokenType.WORD)
|
||||||
@@ -1110,9 +1123,96 @@ class PubmedQueryParser:
|
|||||||
|
|
||||||
# Non-date range or no field → plain text
|
# Non-date range or no field → plain text
|
||||||
txt = f"{start_val}:{end_val}"
|
txt = f"{start_val}:{end_val}"
|
||||||
if field:
|
# R29: field tag is already stored in Term.field — do not append to text
|
||||||
txt = f"{txt}[{field}]"
|
return [Term(txt, field=field or "", is_not=False)]
|
||||||
return [Term(txt, field=field, is_not=False)]
|
|
||||||
|
# ─── R29: Open-ended date range helper ───
|
||||||
|
def _handle_date_range_edge(self, result: ParsedPubmedQuery, negated: bool = False,
|
||||||
|
start_val: str | None = None, end_val: str | None = None) -> list[Term]:
|
||||||
|
"""Handle open-ended ranges like :2024[DP] or 2024:[DP]."""
|
||||||
|
field = None
|
||||||
|
if self.peek().type == TokenType.FIELD:
|
||||||
|
ft = self.advance()
|
||||||
|
field = ft.value[1:-1].upper()
|
||||||
|
_norm = _normalize_field_label(field)
|
||||||
|
if _norm is not None:
|
||||||
|
field = _norm
|
||||||
|
|
||||||
|
if field not in _DATE_RANGE_FIELDS:
|
||||||
|
txt = f"{start_val or ''}:{end_val or ''}"
|
||||||
|
return [Term(txt, field=field or "", is_not=False)]
|
||||||
|
|
||||||
|
attr_map = {
|
||||||
|
"DP": ("date_from", "date_to", "year_from", "year_to", "__RANGE_DP__"),
|
||||||
|
"EDAT": ("edat_from", "edat_to", None, None, "__RANGE_EDAT__"),
|
||||||
|
"CRDT": ("crdt_from", "crdt_to", None, None, "__RANGE_CRDT__"),
|
||||||
|
"MHDA": ("mhda_from", "mhda_to", None, None, "__RANGE_MHDA__"),
|
||||||
|
"LR": ("lr_from", "lr_to", None, None, "__RANGE_LR__"),
|
||||||
|
"DCOM": ("dcom_from", "dcom_to", None, None, "__RANGE_DCOM__"),
|
||||||
|
"DEP": ("dep_from", "dep_to", None, None, "__RANGE_DEP__"),
|
||||||
|
}
|
||||||
|
date_attr, date_attr_to, yr_from_attr, yr_to_attr, marker_field = attr_map[field]
|
||||||
|
|
||||||
|
if self._depth == 0:
|
||||||
|
if start_val is not None and end_val is None:
|
||||||
|
# Open-ended end: 2024:[DP] → from start_val onwards
|
||||||
|
if start_val.isdigit() and len(start_val) == 4:
|
||||||
|
if yr_from_attr is not None:
|
||||||
|
if negated:
|
||||||
|
result._neg_single_dates.setdefault(field, []).append(
|
||||||
|
(f"{start_val}-01-01", None))
|
||||||
|
else:
|
||||||
|
curr = getattr(result, yr_from_attr)
|
||||||
|
v = int(start_val)
|
||||||
|
setattr(result, yr_from_attr, max(curr, v) if curr is not None else v)
|
||||||
|
else:
|
||||||
|
if negated:
|
||||||
|
result._neg_single_dates.setdefault(field, []).append(
|
||||||
|
(f"{start_val}-01-01", None))
|
||||||
|
else:
|
||||||
|
curr = getattr(result, date_attr)
|
||||||
|
v = f"{start_val}-01-01"
|
||||||
|
setattr(result, date_attr, max(curr, v) if curr is not None else v)
|
||||||
|
else:
|
||||||
|
if _PARTIAL_DATE_RE.match(start_val):
|
||||||
|
start_val, _ = _expand_partial_date(start_val)
|
||||||
|
if negated:
|
||||||
|
result._neg_single_dates.setdefault(field, []).append((start_val, None))
|
||||||
|
else:
|
||||||
|
curr = getattr(result, date_attr)
|
||||||
|
setattr(result, date_attr, max(curr, start_val) if curr is not None else start_val)
|
||||||
|
elif end_val is not None and start_val is None:
|
||||||
|
# Open-ended start: :2024[DP] → up to end_val
|
||||||
|
if end_val.isdigit() and len(end_val) == 4:
|
||||||
|
if yr_to_attr is not None:
|
||||||
|
if negated:
|
||||||
|
result._neg_single_dates.setdefault(field, []).append(
|
||||||
|
(None, f"{end_val}-12-31"))
|
||||||
|
else:
|
||||||
|
curr = getattr(result, yr_to_attr)
|
||||||
|
v = int(end_val)
|
||||||
|
setattr(result, yr_to_attr, min(curr, v) if curr is not None else v)
|
||||||
|
else:
|
||||||
|
if negated:
|
||||||
|
result._neg_single_dates.setdefault(field, []).append(
|
||||||
|
(None, f"{end_val}-12-31"))
|
||||||
|
else:
|
||||||
|
curr = getattr(result, date_attr_to)
|
||||||
|
v = f"{end_val}-12-31"
|
||||||
|
setattr(result, date_attr_to, min(curr, v) if curr is not None else v)
|
||||||
|
else:
|
||||||
|
if _PARTIAL_DATE_RE.match(end_val):
|
||||||
|
_, end_val = _expand_partial_date(end_val)
|
||||||
|
if negated:
|
||||||
|
result._neg_single_dates.setdefault(field, []).append((None, end_val))
|
||||||
|
else:
|
||||||
|
curr = getattr(result, date_attr_to)
|
||||||
|
setattr(result, date_attr_to, min(curr, end_val) if curr is not None else end_val)
|
||||||
|
|
||||||
|
marker = Term(f"{start_val or ''}:{end_val or ''}", field=marker_field, is_not=False)
|
||||||
|
marker._is_range_end = True
|
||||||
|
result._date_range_markers.append(marker)
|
||||||
|
return [marker]
|
||||||
|
|
||||||
|
|
||||||
# ─── Public API ───
|
# ─── Public API ───
|
||||||
@@ -1198,7 +1298,7 @@ def extract_pubmed_query_for_prisma(query: str) -> tuple[str, list[str]]:
|
|||||||
|
|
||||||
# P5: Handle field tags with '/' (e.g. Title/Article) or special chars
|
# P5: Handle field tags with '/' (e.g. Title/Article) or special chars
|
||||||
normalized = re.sub(
|
normalized = re.sub(
|
||||||
r'\[([\w/:]+)\]',
|
r'\[([\w/: -]+)\]',
|
||||||
lambda m: f'[{m.group(1).upper()}]',
|
lambda m: f'[{m.group(1).upper()}]',
|
||||||
query,
|
query,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -560,7 +560,7 @@ class AdvancedSearchEngine:
|
|||||||
GlobalLiterature.abstract.isnot(None),
|
GlobalLiterature.abstract.isnot(None),
|
||||||
GlobalLiterature.abstract != '',
|
GlobalLiterature.abstract != '',
|
||||||
))
|
))
|
||||||
if is_free_full_text and is_oa is None:
|
if is_free_full_text:
|
||||||
conditions.append(GlobalLiterature.is_oa == True)
|
conditions.append(GlobalLiterature.is_oa == True)
|
||||||
if has_full_text:
|
if has_full_text:
|
||||||
conditions.append(GlobalLiterature.pmc_id.isnot(None))
|
conditions.append(GlobalLiterature.pmc_id.isnot(None))
|
||||||
@@ -865,6 +865,19 @@ class AdvancedSearchEngine:
|
|||||||
"pages": pp.pages_terms,
|
"pages": pp.pages_terms,
|
||||||
"lid": pp.lid_terms,
|
"lid": pp.lid_terms,
|
||||||
}
|
}
|
||||||
|
_NULL_SAFE_COL_MAP = {
|
||||||
|
"title": GlobalLiterature.title,
|
||||||
|
"abstract": GlobalLiterature.abstract,
|
||||||
|
"author": GlobalLiterature.author_names_text,
|
||||||
|
"journal": GlobalLiterature.journal,
|
||||||
|
"affiliation": GlobalLiterature.authors,
|
||||||
|
"language": GlobalLiterature.language,
|
||||||
|
"volume": GlobalLiterature.volume,
|
||||||
|
"issue": GlobalLiterature.issue,
|
||||||
|
"pages": GlobalLiterature.pages,
|
||||||
|
"lid": GlobalLiterature.doi,
|
||||||
|
"all": GlobalLiterature.title,
|
||||||
|
}
|
||||||
for fld, terms in field_map.items():
|
for fld, terms in field_map.items():
|
||||||
if not terms:
|
if not terms:
|
||||||
continue
|
continue
|
||||||
@@ -873,6 +886,10 @@ class AdvancedSearchEngine:
|
|||||||
for term in terms:
|
for term in terms:
|
||||||
cond = AdvancedSearchEngine._field_condition(fld, term.text, term.exact)
|
cond = AdvancedSearchEngine._field_condition(fld, term.text, term.exact)
|
||||||
if term.is_not:
|
if term.is_not:
|
||||||
|
_ns_col = _NULL_SAFE_COL_MAP.get(fld)
|
||||||
|
if _ns_col is not None:
|
||||||
|
neg_conds.append(or_(not_(cond), _ns_col.is_(None)))
|
||||||
|
else:
|
||||||
neg_conds.append(not_(cond))
|
neg_conds.append(not_(cond))
|
||||||
else:
|
else:
|
||||||
pos_conds.append(cond)
|
pos_conds.append(cond)
|
||||||
@@ -1311,6 +1328,31 @@ class AdvancedSearchEngine:
|
|||||||
child_neg = []
|
child_neg = []
|
||||||
for t in child_group:
|
for t in child_group:
|
||||||
if getattr(t, '_is_range_end', False):
|
if getattr(t, '_is_range_end', False):
|
||||||
|
# R29: process date range markers in child sub-groups
|
||||||
|
_c_marker_cond = AdvancedSearchEngine._build_marker_condition(t)
|
||||||
|
if _c_marker_cond is not None:
|
||||||
|
if getattr(t, 'is_not', False):
|
||||||
|
_cmf = t.field or ""
|
||||||
|
_ctag = _cmf[8:-2] if _cmf.startswith("__RANGE_") and _cmf.endswith("__") else None
|
||||||
|
if _ctag == "DP":
|
||||||
|
child_neg.append(or_(not_(_c_marker_cond), GlobalLiterature.pub_year.is_(None), GlobalLiterature.pub_date.is_(None)))
|
||||||
|
elif _ctag in ("EDAT", "CRDT", "MHDA", "LR", "DCOM", "DEP"):
|
||||||
|
_ctag_col = {
|
||||||
|
"EDAT": GlobalLiterature.entrez_date,
|
||||||
|
"CRDT": GlobalLiterature.create_date,
|
||||||
|
"MHDA": GlobalLiterature.meshed_date,
|
||||||
|
"LR": GlobalLiterature.pubmed_revised,
|
||||||
|
"DCOM": GlobalLiterature.date_completed,
|
||||||
|
"DEP": GlobalLiterature.pub_date,
|
||||||
|
}.get(_ctag)
|
||||||
|
if _ctag_col:
|
||||||
|
child_neg.append(or_(not_(_c_marker_cond), _ctag_col.is_(None)))
|
||||||
|
else:
|
||||||
|
child_neg.append(not_(_c_marker_cond))
|
||||||
|
else:
|
||||||
|
child_neg.append(not_(_c_marker_cond))
|
||||||
|
else:
|
||||||
|
child_pos.append(_c_marker_cond)
|
||||||
continue
|
continue
|
||||||
cond = await AdvancedSearchEngine._single_term_condition(db, t)
|
cond = await AdvancedSearchEngine._single_term_condition(db, t)
|
||||||
if cond is not None:
|
if cond is not None:
|
||||||
@@ -1691,7 +1733,7 @@ class AdvancedSearchEngine:
|
|||||||
from datetime import date as _d
|
from datetime import date as _d
|
||||||
return and_(col >= _d(year, 1, 1), col <= _d(year, 12, 31))
|
return and_(col >= _d(year, 1, 1), col <= _d(year, 12, 31))
|
||||||
# R28: partial date YYYY-MM inside group
|
# R28: partial date YYYY-MM inside group
|
||||||
from app.services.pubmed_query_parser import _PARTIAL_DATE_RE as _PDR
|
from app.services.pubmed_query_parser import _PARTIAL_DATE_RE as _PDR, _expand_partial_date
|
||||||
if _PDR.match(_term_text):
|
if _PDR.match(_term_text):
|
||||||
_df, _dt = _expand_partial_date(_term_text)
|
_df, _dt = _expand_partial_date(_term_text)
|
||||||
from datetime import date as _d
|
from datetime import date as _d
|
||||||
@@ -1789,8 +1831,11 @@ class AdvancedSearchEngine:
|
|||||||
if exact and not _wildcard:
|
if exact and not _wildcard:
|
||||||
# P4: 精确短语 → phraseto_tsquery(利用 GIN 索引,保留词序)
|
# P4: 精确短语 → phraseto_tsquery(利用 GIN 索引,保留词序)
|
||||||
# P13: 加入 journal/journal_iso ILIKE(不在 tsvector 中)
|
# P13: 加入 journal/journal_iso ILIKE(不在 tsvector 中)
|
||||||
|
# R29: 加入 title/abstract ILIKE(非英文如中文无法被 tsquery 匹配)
|
||||||
return or_(
|
return or_(
|
||||||
GlobalLiterature.search_tsv.op("@@")(func.phraseto_tsquery("english", term)),
|
GlobalLiterature.search_tsv.op("@@")(func.phraseto_tsquery("english", term)),
|
||||||
|
GlobalLiterature.title.ilike(pat),
|
||||||
|
GlobalLiterature.abstract.ilike(pat),
|
||||||
GlobalLiterature.journal.ilike(pat),
|
GlobalLiterature.journal.ilike(pat),
|
||||||
GlobalLiterature.journal_iso.ilike(pat),
|
GlobalLiterature.journal_iso.ilike(pat),
|
||||||
text("EXISTS (SELECT 1 FROM jsonb_array_elements(global_literature.authors) AS _e "
|
text("EXISTS (SELECT 1 FROM jsonb_array_elements(global_literature.authors) AS _e "
|
||||||
|
|||||||
@@ -2155,3 +2155,85 @@ if _PARTIAL_DATE_RE.match(end_val):
|
|||||||
- ✅ **1007 tests passed, 0 failed**
|
- ✅ **1007 tests passed, 0 failed**
|
||||||
- 全部搜索测试通过(parser 36 + search engine 集成测试)
|
- 全部搜索测试通过(parser 36 + search engine 集成测试)
|
||||||
- 无回归
|
- 无回归
|
||||||
|
|
||||||
|
## R29 (2026-07-29): 第四轮并行审计修复
|
||||||
|
|
||||||
|
### 审计范围
|
||||||
|
|
||||||
|
4 路并行 agent,分别覆盖:
|
||||||
|
1. `pubmed_query_parser.py` 语法正确性(递归下降解析器)
|
||||||
|
2. `search_engine.py` 条件构建(_pubmed_conditions 全字段路径)
|
||||||
|
3. 纯文本降级路径(_is_flat_text 决策、缓存键、引号短语、facet 一致性)
|
||||||
|
4. 前端搜索参数映射(前后端参数完整性、URL 同步、分页)
|
||||||
|
|
||||||
|
### R29-1 (P0): `_expand_partial_date` 未导入
|
||||||
|
|
||||||
|
**文件**:[search_engine.py:1696](backend/app/services/search_engine.py#L1696)
|
||||||
|
|
||||||
|
**根因**:`_single_term_condition()` 的 YYYY-MM 部分日期分支调用 `_expand_partial_date()` 但该函数未从 `pubmed_query_parser` 导入。导入语句只导入了 `_PARTIAL_DATE_RE` 和 `_validate_date_str`。
|
||||||
|
|
||||||
|
**影响**:组内 YYYY-MM 日期(如 `(2024-06[DP] OR cancer)`)触发 `NameError`,被 `except Exception` 吞没后降级到纯文本。
|
||||||
|
|
||||||
|
**修复**:添加 `_expand_partial_date` 到 inline import。
|
||||||
|
|
||||||
|
### R29-2 (P0): 非日期范围 field 标签混入文本
|
||||||
|
|
||||||
|
**文件**:[pubmed_query_parser.py:1111-1115](backend/app/services/pubmed_query_parser.py#L1111)
|
||||||
|
|
||||||
|
**根因**:`_parse_range()` 对非日期范围(如 `term1:term2[MH]`)的 fallthrough 路径将 `[MH]` 追加入 term 文本,导致 `text="term1:term2[MH]"` 而 `Term.field="MH"` 已正确设置。引擎按 `field` 路由(正确到 MeSH 路径),但 `text` 中的 `[MH]` 被作为字面搜索词。
|
||||||
|
|
||||||
|
**修复**:去除 `[{field}]` 文本拼接,只保留 `start_val:end_val`。
|
||||||
|
|
||||||
|
### R29-3 (P0): 中文精确短语缺少 title/abstract ILIKE
|
||||||
|
|
||||||
|
**文件**:[search_engine.py:1789-1798](backend/app/services/search_engine.py#L1789)
|
||||||
|
|
||||||
|
**根因**:`_field_condition("all", exact=True)` 分支使用 `phraseto_tsquery('english', ...)` 无法匹配非英文文本(如中文)。缺少 title/abstract ILIKE 回退。非精确分支(line 1832)正确包含这些 ILIKE。
|
||||||
|
|
||||||
|
**影响**:`"肺癌"[TIAB]` 等中文精确短语返回零结果。
|
||||||
|
|
||||||
|
**修复**:在 exact 分支中添加 `title.ilike(pat)` 和 `abstract.ilike(pat)`。
|
||||||
|
|
||||||
|
### R29-4 (P1): 子组日期范围标记被丢弃
|
||||||
|
|
||||||
|
**文件**:[search_engine.py:1313-1314](backend/app/services/search_engine.py#L1313)
|
||||||
|
|
||||||
|
**根因**:子组(sub-group,由 `_parse_or_expr` AND 集群 OR 拆分产生)的日期范围标记在 child processing 循环中被 `continue` 跳过。
|
||||||
|
|
||||||
|
**影响**:`(A OR B AND 2020:2024[DP])` 等查询中日期范围丢失。
|
||||||
|
|
||||||
|
**修复**:子组循环中添加 `_build_marker_condition` + NULL 安全 NOT,逻辑与父组一致。
|
||||||
|
|
||||||
|
### R29-5 (P1): 开区间日期范围不支持
|
||||||
|
|
||||||
|
**文件**:[pubmed_query_parser.py:930-938](backend/app/services/pubmed_query_parser.py#L930)
|
||||||
|
|
||||||
|
**根因**:`:2024[DP]`(截止到 2024)和 `2024:[DP]`(从 2024 起)的 COLON 在首个位置或无第二个 NUMBER 的模式不被范围检测匹配。
|
||||||
|
|
||||||
|
**修复**:在 `_parse_atom` 中增加 `:NUMBER[FIELD]` 和 `NUMBER:[FIELD]` 检测,路由到新方法 `_handle_date_range_edge()`,正确设置全局属性、标记和 `_neg_single_dates`。
|
||||||
|
|
||||||
|
### R29-6 (P2): field_map NOT 条件缺少 NULL 安全
|
||||||
|
|
||||||
|
**文件**:[search_engine.py:874-881](backend/app/services/search_engine.py#L874)
|
||||||
|
|
||||||
|
**根因**:`[TI]`/`[AB]`/`[AU]` 等字段标签的否定词使用裸 `not_(cond)`,导致字段为 NULL 的行被错误排除。
|
||||||
|
|
||||||
|
**修复**:添加 `_NULL_SAFE_COL_MAP`,对每个字段的否定条件使用 `or_(not_(cond), col.is_(None))`。
|
||||||
|
|
||||||
|
### R29-7 (LOW): PRISMA 导出正则不匹配含空格的字段标签
|
||||||
|
|
||||||
|
**文件**:[pubmed_query_parser.py:1301](backend/app/services/pubmed_query_parser.py#L1301)
|
||||||
|
|
||||||
|
**修复**:`[\w/:]+` → `[\w/: -]+` 以匹配 `[MeSH Terms]`、`[Date - Publication]` 等。
|
||||||
|
|
||||||
|
### R29-8 (LOW): `is_free_full_text` 在 `is_oa` 已设时被跳过
|
||||||
|
|
||||||
|
**文件**:[search_engine.py:563](backend/app/services/search_engine.py#L563)
|
||||||
|
|
||||||
|
**修复**:移除 `is_oa is None` 守卫,`is_free_full_text` 始终独立生效。
|
||||||
|
|
||||||
|
## 验证
|
||||||
|
|
||||||
|
- ✅ **1007 tests passed, 0 failed**
|
||||||
|
- 全部搜索测试通过
|
||||||
|
- 无回归
|
||||||
|
|||||||
Reference in New Issue
Block a user