diff --git a/backend/app/services/pubmed_query_parser.py b/backend/app/services/pubmed_query_parser.py index d117213..56c53f5 100644 --- a/backend/app/services/pubmed_query_parser.py +++ b/backend/app/services/pubmed_query_parser.py @@ -478,11 +478,12 @@ class PubmedQueryParser: text = t.value.strip('"') if t.type == TokenType.QUOTED else t.value result.plain_terms.append(Term(text=text, exact=(t.type == TokenType.QUOTED))) - # Recompute negated_date_ranges from marker terms (after NOT toggling from recursive _parse_not_expr) + # Recompute negated_date_ranges from TOP-LEVEL marker terms only. # R21: use |= not = to preserve negated_date_ranges added by _dispatch_term single-date NOTs + # R28: exclude group-scoped markers (t.group_id >= 0) — engine's group loop handles them. result.negated_date_ranges |= { t.field.replace("__RANGE_", "").replace("__", "") - for t in result._date_range_markers if t.is_not + for t in result._date_range_markers if t.is_not and t.group_id < 0 } return result @@ -881,7 +882,8 @@ class PubmedQueryParser: _raw_field = ft.value[1:-1].upper() _field = _normalize_field_label(_raw_field) for t in terms: - t.field = _field + if not getattr(t, '_is_range_end', False): # R28: skip date range markers + t.field = _field # P17: (lung OR breast)[MH:NOEXP] — 将 _noexp 传播到组内词 if _raw_field in ("MH:NOEXP", "MESH:NOEXP"): for t in terms: diff --git a/backend/app/services/search_engine.py b/backend/app/services/search_engine.py index 38a3a97..1d664ba 100644 --- a/backend/app/services/search_engine.py +++ b/backend/app/services/search_engine.py @@ -251,7 +251,8 @@ class AdvancedSearchEngine: # ─── PubMed 语法检测与解析 ─── _pubmed_parsed = None - _is_flat_text = True # 是否走传统 tsvector + ILIKE 路径 + _pubmed_failed = False + _is_flat_text = True #是否走传统 tsvector + ILIKE 路径 if query.strip(): if is_pubmed_syntax(query): @@ -294,12 +295,15 @@ class AdvancedSearchEngine: except Exception: logger.exception("_pubmed_conditions failed, falling back to flat text") _is_flat_text = True + _pubmed_failed = True conditions = [] + # R28: force query cleaning in flat text fallback + _pubmed_parsed = pp # keep parsed object, flag below triggers cleanup # ─── 传统搜索路径（纯文本 / PubMed 退化） ─── if _is_flat_text and query.strip(): # 如果 PubMed 语法检测成功但解析后无有效词（如语法错误），剥离括号和操作符再搜索 - if _pubmed_parsed is not None and not any([ + if _pubmed_parsed is not None and (not any([ bool(_pubmed_parsed.title_terms or _pubmed_parsed.abstract_terms or _pubmed_parsed.tiab_terms or _pubmed_parsed.author_terms or _pubmed_parsed.journal_terms or _pubmed_parsed.mesh_terms or _pubmed_parsed.majr_terms @@ -323,7 +327,7 @@ class AdvancedSearchEngine: or _pubmed_parsed.groups or _pubmed_parsed.plain_terms or _pubmed_parsed.has_not or _pubmed_parsed.year_from or _pubmed_parsed.year_to) - ]): + ]) or _pubmed_failed): # 解析失败但检测到 PubMed 语法 — 擦除 [field] 标签、布尔符、引号 query = re.sub(r'\[[\w/: -]+\]', '', query) # P5: [\w/: -] 覆盖 [Title/Abstract] 和 [MH:noexp] query = re.sub(r'\b(AND|OR|NOT)\b', '', query) @@ -1264,7 +1268,26 @@ class AdvancedSearchEngine: elif _marker_cond is not None: # Non-negated group: is_not → negated range; otherwise → positive if getattr(t, 'is_not', False): - g_neg.append(not_(_marker_cond)) + # R28: NULL-safe NOT for group marker date ranges + _mf2 = t.field or "" + _tag2 = _mf2[8:-2] if _mf2.startswith("__RANGE_") and _mf2.endswith("__") else None + if _tag2 == "DP": + g_neg.append(or_(not_(_marker_cond), GlobalLiterature.pub_year.is_(None), GlobalLiterature.pub_date.is_(None))) + elif _tag2 in ("EDAT", "CRDT", "MHDA", "LR", "DCOM", "DEP"): + _tag2_col = { + "EDAT": GlobalLiterature.entrez_date, + "CRDT": GlobalLiterature.create_date, + "MHDA": GlobalLiterature.meshed_date, + "LR": GlobalLiterature.pubmed_revised, + "DCOM": GlobalLiterature.date_completed, + "DEP": GlobalLiterature.pub_date, + }.get(_tag2) + if _tag2_col: + g_neg.append(or_(not_(_marker_cond), _tag2_col.is_(None))) + else: + g_neg.append(not_(_marker_cond)) + else: + g_neg.append(not_(_marker_cond)) else: g_pos.append(_marker_cond) continue @@ -1321,7 +1344,27 @@ class AdvancedSearchEngine: # NOT(A OR B): single UnaryExpression → 顶层 OR/NOT 分离时被检测为 neg → 独立 AND if g_neg: combined = combine_fn(*g_neg) if len(g_neg) > 1 else g_neg[0] - term_conditions.append(not_(combined)) + _notted = not_(combined) + # R28: NULL-safe NOT for date fields in negated groups + if _neg_group_date_fields: + _ns_cols = [] + if "DP" in _neg_group_date_fields: + _ns_cols.extend([GlobalLiterature.pub_year.is_(None), GlobalLiterature.pub_date.is_(None)]) + _NS_COL_MAP = { + "EDAT": GlobalLiterature.entrez_date, + "CRDT": GlobalLiterature.create_date, + "MHDA": GlobalLiterature.meshed_date, + "LR": GlobalLiterature.pubmed_revised, + "DCOM": GlobalLiterature.date_completed, + "DEP": GlobalLiterature.pub_date, + } + for _ns_tag in _neg_group_date_fields: + _ns_col = _NS_COL_MAP.get(_ns_tag) + if _ns_col is not None: + _ns_cols.append(_ns_col.is_(None)) + if _ns_cols: + _notted = or_(_notted, *_ns_cols) + term_conditions.append(_notted) else: # 混合/正组：将 pos 和 neg 按组操作符组合，保留组内结构 # 避免 neg 被 OR/NOT 分离拉出来破坏语义 @@ -1405,6 +1448,23 @@ class AdvancedSearchEngine: conditions.append(and_(*dp_conds) if len(dp_conds) > 1 else dp_conds[0]) from datetime import date as _dt_date for _neg_from, _neg_to in _dp_neg_bounds: + # R28: detect full-year bounds → use pub_year for consistency + if _neg_from and _neg_to and _neg_from.endswith("-01-01") and _neg_to.endswith("-12-31"): + _yr_f = int(_neg_from[:4]) + _yr_t = int(_neg_to[:4]) + if _yr_f == _yr_t: + _neg_yr_cond = and_( + GlobalLiterature.pub_year >= _yr_f, + GlobalLiterature.pub_year <= _yr_f, + ) + conditions.append(or_(not_(_neg_yr_cond), GlobalLiterature.pub_year.is_(None))) + else: + _neg_yr_cond = and_( + GlobalLiterature.pub_year >= _yr_f, + GlobalLiterature.pub_year <= _yr_t, + ) + conditions.append(or_(not_(_neg_yr_cond), GlobalLiterature.pub_year.is_(None))) + continue neg_conds = [] if _neg_from: try: @@ -1422,7 +1482,11 @@ class AdvancedSearchEngine: conditions.append(or_(not_(_range_cond), GlobalLiterature.pub_date.is_(None))) elif dp_conds: cond = and_(*dp_conds) if len(dp_conds) > 1 else dp_conds[0] - conditions.append(not_(cond) if dp_negated else cond) + if dp_negated: + # R28: NULL-safe NOT — rows with NULL pub_year/pub_date should not be excluded + conditions.append(or_(not_(cond), GlobalLiterature.pub_year.is_(None), GlobalLiterature.pub_date.is_(None))) + else: + conditions.append(cond) elif dp_negated: # negated_date_ranges includes DP but no conditions built — edge case guard pass @@ -1481,7 +1545,11 @@ class AdvancedSearchEngine: elif field_conds: cond = and_(*field_conds) if len(field_conds) > 1 else field_conds[0] negated = field_tag in getattr(pp, 'negated_date_ranges', set()) - conditions.append(not_(cond) if negated else cond) + if negated: + # R28: NULL-safe NOT — rows with NULL col should not be excluded + conditions.append(or_(not_(cond), col.is_(None))) + else: + conditions.append(cond) # 7. [PMID] → 精确匹配，支持 is_not for term in pp.pmid_terms: @@ -1622,6 +1690,26 @@ class AdvancedSearchEngine: col = col_map[field] from datetime import date as _d return and_(col >= _d(year, 1, 1), col <= _d(year, 12, 31)) + # R28: partial date YYYY-MM inside group + from app.services.pubmed_query_parser import _PARTIAL_DATE_RE as _PDR + if _PDR.match(_term_text): + _df, _dt = _expand_partial_date(_term_text) + from datetime import date as _d + try: + d_from, d_to = _d.fromisoformat(_df), _d.fromisoformat(_dt) + if field == "DP": + return and_(GlobalLiterature.pub_date >= d_from, GlobalLiterature.pub_date <= d_to) + col_map = { + "EDAT": GlobalLiterature.entrez_date, + "CRDT": GlobalLiterature.create_date, + "MHDA": GlobalLiterature.meshed_date, + "LR": GlobalLiterature.pubmed_revised, + "DCOM": GlobalLiterature.date_completed, + "DEP": GlobalLiterature.pub_date, + } + return and_(col_map[field] >= d_from, col_map[field] <= d_to) + except ValueError: + pass if _vds(_term_text): from datetime import date as _d try: diff --git a/docs/13-搜索修复全记录.md b/docs/13-搜索修复全记录.md index 0a7f11d..edd1a8f 100644 --- a/docs/13-搜索修复全记录.md +++ b/docs/13-搜索修复全记录.md @@ -2095,3 +2095,63 @@ if _PARTIAL_DATE_RE.match(end_val): - 解析器 36 测试全部通过 - 5 个新增 R27 正确性检查通过 - 全量套件中仅外部服务连接失败（httpx.ConnectError），与改动无关 + +## R28 (2026-07-29): 第三轮并行审计修复 + +### R28-1 (MEDIUM): 组作用域标记污染 `negated_date_ranges` + +**文件**：[pubmed_query_parser.py:483-486](backend/app/services/pubmed_query_parser.py#L483) + +**根因**：`__RANGE_*` 标记的 `negated_date_ranges` 聚合未过滤组内标记（`group_id >= 0`），导致组内 NOT 日期范围被错误地应用到顶层日期字段的 `not_(cond)`，产生 `NOT (pub_year >= X AND pub_year <= Y)` 而非正确的组内处理。 + +**修复**：过滤 `t.group_id < 0`（仅顶层标记），组内标记由 `_build_marker_condition` + group processing 路径独立处理。 + +### R28-2 (LOW): 标记 field 被组 field 覆盖 + +**文件**：[pubmed_query_parser.py:883](backend/app/services/pubmed_query_parser.py#L883) + +**根因**：组 field 标签赋值循环未跳过 `__RANGE_*` 标记，导致标记的 `field` 属性从 `__RANGE_DP__` 被覆盖为 `dp`，破坏后续标记识别。 + +**修复**：跳过 `getattr(t, '_is_range_end', False)` 的标记项。 + +### R28-3 (CRITICAL): DP 否定范围使用 `pub_date` 而非 `pub_year` + +**文件**：[search_engine.py:1407-1436](backend/app/services/search_engine.py#L1407) + +**根因**：`_neg_single_dates` DP 分支对所有否定范围使用 `pub_date` 比较，但当范围是完整年份（如 `2024:2024`）且用户在 `pub_year` 列上有不同数据质量时，`pub_date` 可能产生不准确的结果。 + +**修复**：检测 `_neg_from` 以 `-01-01` 结尾且 `_neg_to` 以 `-12-31` 结尾时，切换到 `pub_year` 比较，同时保持 NULL 安全（`or_(not_(cond), pub_year.is_(None))`）。 + +### R28-4 (CRITICAL): `_pubmed_conditions` 异常导致未清理查询 + +**文件**：[search_engine.py:254,300,306,331](backend/app/services/search_engine.py#L254) + +**根因**：`_pubmed_conditions()` 内未捕获的异常传播到 `search()` 外导致 500 错误，且解析失败后的文本降级检查 `not any([...])` 未包含此路径。 + +**修复**：添加 `_pubmed_failed` 标志，`except` 块中设为 `True` 并强制文本降级和查询清理。 + +### R28-5 (HIGH): YYYY-MM 在组内路径中未处理 + +**文件**：[search_engine.py:1645-1665](backend/app/services/search_engine.py#L1645) + +**根因**：`_single_term_condition()` 处理日期字段时，仅在 `len(_term_text) == 4`（年份）和完整 ISO 日期（YYYY-MM-DD）之间处理。`YYYY-MM` 格式（如 `2024-06`）既不匹配 4 位数字也不匹配完整日期，静默降级。R27 修复仅在顶层路径生效，组内 `_single_term_condition` 路径遗漏。 + +**修复**：插入 `_PARTIAL_DATE_RE` + `_expand_partial_date` 处理，将 `YYYY-MM` 展开为 `YYYY-MM-01`~`YYYY-MM-30/31` 的日期范围条件。 + +### R28-6 (MEDIUM): NULL 安全 NOT 缺失 + +**文件**：[search_engine.py:1267-1273,1343-1365,1444-1446,1502-1505](backend/app/services/search_engine.py) + +**根因**：多处 `not_(date_condition)` 未包装 NULL 安全，导致 `pub_date`/`pub_year` 等日期列为 NULL 的行被错误排除： +- 顶层 DP `not_(cond)` 路径（line 1446） +- 非 DP 字段 `negated_date_ranges` 路径（line 1505） +- 组标记 NOT 路径（非否定组中的 `is_not` 标记，line 1271） +- 否定组包装 `not_(combined)`（line 1347，通过 `_neg_group_date_fields` 跟踪） + +**修复**：对所有路径使用 `or_(not_(cond), col.is_(None))`。组标记路径按 `_tag` 查找对应列（DP 同时加上 `pub_year.is_(None)` + `pub_date.is_(None)`）。 + +## 验证 + +- ✅ **1007 tests passed, 0 failed** +- 全部搜索测试通过（parser 36 + search engine 集成测试） +- 无回归