scilit/backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 第29轮搜索审计修复 — 缺失导入/中文精确短语ILIKE/子组日期标记/开区间范围/NULL安全等8项
CI / backend (push) Canceled after 0s
Details
CI / frontend (push) Canceled after 0s
Details

Browse Source
This commit is contained in:
34047007@qq.com
2026-07-29 06:16:11 +08:00
parent 2b085d3445
commit 475d6bb80c
3 changed files with 234 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/pubmed_query_parser.py
+104 -4
View File
@@ -932,6 +932,19 @@ class PubmedQueryParser:
t1 = self.peek_n(1)
t2 = self.peek_n(2)
# R29: open-ended start range :2024[DP] → COLON NUMBER/DATE FIELD
if t0.type == TokenType.COLON and t1 is not None and t1.type in (TokenType.NUMBER, TokenType.DATE):
self.advance() # consume COLON
e_val = self.advance().value # end value
return self._handle_date_range_edge(result, negated, start_val=None, end_val=e_val)
# R29: open-ended end range 2024:[DP] → NUMBER/DATE COLON FIELD
if (t0.type in (TokenType.NUMBER, TokenType.DATE) and t1 is not None and t1.type == TokenType.COLON
and t2 is not None and t2.type == TokenType.FIELD):
s_val = self.advance().value # start value
self.advance() # consume COLON
return self._handle_date_range_edge(result, negated, start_val=s_val, end_val=None)
if (t1 is not None and t1.type == TokenType.COLON
and t2 is not None
and t0.type in (TokenType.NUMBER, TokenType.DATE, TokenType.WORD)
@@ -1110,9 +1123,96 @@ class PubmedQueryParser:
# Non-date range or no field → plain text
txt = f"{start_val}:{end_val}"
if field:
txt = f"{txt}[{field}]"
return [Term(txt, field=field, is_not=False)]
# R29: field tag is already stored in Term.field — do not append to text
return [Term(txt, field=field or "", is_not=False)]
# ─── R29: Open-ended date range helper ───
def _handle_date_range_edge(self, result: ParsedPubmedQuery, negated: bool = False,
start_val: str | None = None, end_val: str | None = None) -> list[Term]:
"""Handle open-ended ranges like :2024[DP] or 2024:[DP]."""
field = None
if self.peek().type == TokenType.FIELD:
ft = self.advance()
field = ft.value[1:-1].upper()
_norm = _normalize_field_label(field)
if _norm is not None:
field = _norm
if field not in _DATE_RANGE_FIELDS:
txt = f"{start_val or ''}:{end_val or ''}"
return [Term(txt, field=field or "", is_not=False)]
attr_map = {
"DP": ("date_from", "date_to", "year_from", "year_to", "__RANGE_DP__"),
"EDAT": ("edat_from", "edat_to", None, None, "__RANGE_EDAT__"),
"CRDT": ("crdt_from", "crdt_to", None, None, "__RANGE_CRDT__"),
"MHDA": ("mhda_from", "mhda_to", None, None, "__RANGE_MHDA__"),
"LR": ("lr_from", "lr_to", None, None, "__RANGE_LR__"),
"DCOM": ("dcom_from", "dcom_to", None, None, "__RANGE_DCOM__"),
"DEP": ("dep_from", "dep_to", None, None, "__RANGE_DEP__"),
}
date_attr, date_attr_to, yr_from_attr, yr_to_attr, marker_field = attr_map[field]
if self._depth == 0:
if start_val is not None and end_val is None:
# Open-ended end: 2024:[DP] → from start_val onwards
if start_val.isdigit() and len(start_val) == 4:
if yr_from_attr is not None:
if negated:
result._neg_single_dates.setdefault(field, []).append(
(f"{start_val}-01-01", None))
else:
curr = getattr(result, yr_from_attr)
v = int(start_val)
setattr(result, yr_from_attr, max(curr, v) if curr is not None else v)
else:
if negated:
result._neg_single_dates.setdefault(field, []).append(
(f"{start_val}-01-01", None))
else:
curr = getattr(result, date_attr)
v = f"{start_val}-01-01"
setattr(result, date_attr, max(curr, v) if curr is not None else v)
else:
if _PARTIAL_DATE_RE.match(start_val):
start_val, _ = _expand_partial_date(start_val)
if negated:
result._neg_single_dates.setdefault(field, []).append((start_val, None))
else:
curr = getattr(result, date_attr)
setattr(result, date_attr, max(curr, start_val) if curr is not None else start_val)
elif end_val is not None and start_val is None:
# Open-ended start: :2024[DP] → up to end_val
if end_val.isdigit() and len(end_val) == 4:
if yr_to_attr is not None:
if negated:
result._neg_single_dates.setdefault(field, []).append(
(None, f"{end_val}-12-31"))
else:
curr = getattr(result, yr_to_attr)
v = int(end_val)
setattr(result, yr_to_attr, min(curr, v) if curr is not None else v)
else:
if negated:
result._neg_single_dates.setdefault(field, []).append(
(None, f"{end_val}-12-31"))
else:
curr = getattr(result, date_attr_to)
v = f"{end_val}-12-31"
setattr(result, date_attr_to, min(curr, v) if curr is not None else v)
else:
if _PARTIAL_DATE_RE.match(end_val):
_, end_val = _expand_partial_date(end_val)
if negated:
result._neg_single_dates.setdefault(field, []).append((None, end_val))
else:
curr = getattr(result, date_attr_to)
setattr(result, date_attr_to, min(curr, end_val) if curr is not None else end_val)
marker = Term(f"{start_val or ''}:{end_val or ''}", field=marker_field, is_not=False)
marker._is_range_end = True
result._date_range_markers.append(marker)
return [marker]
# ─── Public API ───
@@ -1198,7 +1298,7 @@ def extract_pubmed_query_for_prisma(query: str) -> tuple[str, list[str]]:
# P5: Handle field tags with '/' (e.g. Title/Article) or special chars
normalized = re.sub(
r'\[([\w/:]+)\]',
r'\[([\w/: -]+)\]',
lambda m: f'[{m.group(1).upper()}]',
query,
)
backend/app/services/search_engine.py
+48 -3
View File
@@ -560,7 +560,7 @@ class AdvancedSearchEngine:
GlobalLiterature.abstract.isnot(None),
GlobalLiterature.abstract != '',
))
if is_free_full_text and is_oa is None:
if is_free_full_text:
conditions.append(GlobalLiterature.is_oa == True)
if has_full_text:
conditions.append(GlobalLiterature.pmc_id.isnot(None))
@@ -865,6 +865,19 @@ class AdvancedSearchEngine:
"pages": pp.pages_terms,
"lid": pp.lid_terms,
}
_NULL_SAFE_COL_MAP = {
"title": GlobalLiterature.title,
"abstract": GlobalLiterature.abstract,
"author": GlobalLiterature.author_names_text,
"journal": GlobalLiterature.journal,
"affiliation": GlobalLiterature.authors,
"language": GlobalLiterature.language,
"volume": GlobalLiterature.volume,
"issue": GlobalLiterature.issue,
"pages": GlobalLiterature.pages,
"lid": GlobalLiterature.doi,
"all": GlobalLiterature.title,
}
for fld, terms in field_map.items():
if not terms:
continue
@@ -873,7 +886,11 @@ class AdvancedSearchEngine:
for term in terms:
cond = AdvancedSearchEngine._field_condition(fld, term.text, term.exact)
if term.is_not:
neg_conds.append(not_(cond))
_ns_col = _NULL_SAFE_COL_MAP.get(fld)
if _ns_col is not None:
neg_conds.append(or_(not_(cond), _ns_col.is_(None)))
else:
neg_conds.append(not_(cond))
else:
pos_conds.append(cond)
if pos_conds:
@@ -1311,6 +1328,31 @@ class AdvancedSearchEngine:
child_neg = []
for t in child_group:
if getattr(t, '_is_range_end', False):
# R29: process date range markers in child sub-groups
_c_marker_cond = AdvancedSearchEngine._build_marker_condition(t)
if _c_marker_cond is not None:
if getattr(t, 'is_not', False):
_cmf = t.field or ""
_ctag = _cmf[8:-2] if _cmf.startswith("__RANGE_") and _cmf.endswith("__") else None
if _ctag == "DP":
child_neg.append(or_(not_(_c_marker_cond), GlobalLiterature.pub_year.is_(None), GlobalLiterature.pub_date.is_(None)))
elif _ctag in ("EDAT", "CRDT", "MHDA", "LR", "DCOM", "DEP"):
_ctag_col = {
"EDAT": GlobalLiterature.entrez_date,
"CRDT": GlobalLiterature.create_date,
"MHDA": GlobalLiterature.meshed_date,
"LR": GlobalLiterature.pubmed_revised,
"DCOM": GlobalLiterature.date_completed,
"DEP": GlobalLiterature.pub_date,
}.get(_ctag)
if _ctag_col:
child_neg.append(or_(not_(_c_marker_cond), _ctag_col.is_(None)))
else:
child_neg.append(not_(_c_marker_cond))
else:
child_neg.append(not_(_c_marker_cond))
else:
child_pos.append(_c_marker_cond)
continue
cond = await AdvancedSearchEngine._single_term_condition(db, t)
if cond is not None:
@@ -1691,7 +1733,7 @@ class AdvancedSearchEngine:
from datetime import date as _d
return and_(col >= _d(year, 1, 1), col <= _d(year, 12, 31))
# R28: partial date YYYY-MM inside group
from app.services.pubmed_query_parser import _PARTIAL_DATE_RE as _PDR
from app.services.pubmed_query_parser import _PARTIAL_DATE_RE as _PDR, _expand_partial_date
if _PDR.match(_term_text):
_df, _dt = _expand_partial_date(_term_text)
from datetime import date as _d
@@ -1789,8 +1831,11 @@ class AdvancedSearchEngine:
if exact and not _wildcard:
# P4: 精确短语 → phraseto_tsquery（利用 GIN 索引，保留词序）
# P13: 加入 journal/journal_iso ILIKE（不在 tsvector 中）
# R29: 加入 title/abstract ILIKE（非英文如中文无法被 tsquery 匹配）
return or_(
GlobalLiterature.search_tsv.op("@@")(func.phraseto_tsquery("english", term)),
GlobalLiterature.title.ilike(pat),
GlobalLiterature.abstract.ilike(pat),
GlobalLiterature.journal.ilike(pat),
GlobalLiterature.journal_iso.ilike(pat),
text("EXISTS (SELECT 1 FROM jsonb_array_elements(global_literature.authors) AS _e "
docs/13-搜索修复全记录.md
+82
View File
@@ -2155,3 +2155,85 @@ if _PARTIAL_DATE_RE.match(end_val):
- ✅ **1007 tests passed, 0 failed**
- 全部搜索测试通过（parser 36 + search engine 集成测试）
- 无回归
## R29 (2026-07-29): 第四轮并行审计修复
### 审计范围
4 路并行 agent，分别覆盖：
1. `pubmed_query_parser.py` 语法正确性（递归下降解析器）
2. `search_engine.py` 条件构建（_pubmed_conditions 全字段路径）
3. 纯文本降级路径（_is_flat_text 决策、缓存键、引号短语、facet 一致性）
4. 前端搜索参数映射（前后端参数完整性、URL 同步、分页）
### R29-1 (P0): `_expand_partial_date` 未导入
**文件**：[search_engine.py:1696](backend/app/services/search_engine.py#L1696)
**根因**：`_single_term_condition()` 的 YYYY-MM 部分日期分支调用 `_expand_partial_date()` 但该函数未从 `pubmed_query_parser` 导入。导入语句只导入了 `_PARTIAL_DATE_RE` 和 `_validate_date_str`。
**影响**：组内 YYYY-MM 日期（如 `(2024-06[DP] OR cancer)`）触发 `NameError`，被 `except Exception` 吞没后降级到纯文本。
**修复**：添加 `_expand_partial_date` 到 inline import。
### R29-2 (P0): 非日期范围 field 标签混入文本
**文件**：[pubmed_query_parser.py:1111-1115](backend/app/services/pubmed_query_parser.py#L1111)
**根因**：`_parse_range()` 对非日期范围（如 `term1:term2[MH]`）的 fallthrough 路径将 `[MH]` 追加入 term 文本，导致 `text="term1:term2[MH]"` 而 `Term.field="MH"` 已正确设置。引擎按 `field` 路由（正确到 MeSH 路径），但 `text` 中的 `[MH]` 被作为字面搜索词。
**修复**：去除 `[{field}]` 文本拼接，只保留 `start_val:end_val`。
### R29-3 (P0): 中文精确短语缺少 title/abstract ILIKE
**文件**：[search_engine.py:1789-1798](backend/app/services/search_engine.py#L1789)
**根因**：`_field_condition("all", exact=True)` 分支使用 `phraseto_tsquery('english', ...)` 无法匹配非英文文本（如中文）。缺少 title/abstract ILIKE 回退。非精确分支（line 1832）正确包含这些 ILIKE。
**影响**：`"肺癌"[TIAB]` 等中文精确短语返回零结果。
**修复**：在 exact 分支中添加 `title.ilike(pat)` 和 `abstract.ilike(pat)`。
### R29-4 (P1): 子组日期范围标记被丢弃
**文件**：[search_engine.py:1313-1314](backend/app/services/search_engine.py#L1313)
**根因**：子组（sub-group，由 `_parse_or_expr` AND 集群 OR 拆分产生）的日期范围标记在 child processing 循环中被 `continue` 跳过。
**影响**：`(A OR B AND 2020:2024[DP])` 等查询中日期范围丢失。
**修复**：子组循环中添加 `_build_marker_condition` + NULL 安全 NOT，逻辑与父组一致。
### R29-5 (P1): 开区间日期范围不支持
**文件**：[pubmed_query_parser.py:930-938](backend/app/services/pubmed_query_parser.py#L930)
**根因**：`:2024[DP]`（截止到 2024）和 `2024:[DP]`（从 2024 起）的 COLON 在首个位置或无第二个 NUMBER 的模式不被范围检测匹配。
**修复**：在 `_parse_atom` 中增加 `:NUMBER[FIELD]` 和 `NUMBER:[FIELD]` 检测，路由到新方法 `_handle_date_range_edge()`，正确设置全局属性、标记和 `_neg_single_dates`。
### R29-6 (P2): field_map NOT 条件缺少 NULL 安全
**文件**：[search_engine.py:874-881](backend/app/services/search_engine.py#L874)
**根因**：`[TI]`/`[AB]`/`[AU]` 等字段标签的否定词使用裸 `not_(cond)`，导致字段为 NULL 的行被错误排除。
**修复**：添加 `_NULL_SAFE_COL_MAP`，对每个字段的否定条件使用 `or_(not_(cond), col.is_(None))`。
### R29-7 (LOW): PRISMA 导出正则不匹配含空格的字段标签
**文件**：[pubmed_query_parser.py:1301](backend/app/services/pubmed_query_parser.py#L1301)
**修复**：`[\w/:]+` → `[\w/: -]+` 以匹配 `[MeSH Terms]`、`[Date - Publication]` 等。
### R29-8 (LOW): `is_free_full_text` 在 `is_oa` 已设时被跳过
**文件**：[search_engine.py:563](backend/app/services/search_engine.py#L563)
**修复**：移除 `is_oa is None` 守卫，`is_free_full_text` 始终独立生效。
## 验证
- ✅ **1007 tests passed, 0 failed**
- 全部搜索测试通过
- 无回归