From d064dcd0caebb18ce1a7d2ddd62e50612570a10c Mon Sep 17 00:00:00 2001 From: "34047007@qq.com" <34047007@qq.com> Date: Tue, 28 Jul 2026 22:56:04 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E7=AC=AC27=E8=BD=AE=E6=90=9C=E7=B4=A2?= =?UTF-8?q?=E5=AE=A1=E8=AE=A1=E4=BF=AE=E5=A4=8D=20=E2=80=94=20=E8=B7=A8?= =?UTF-8?q?=E5=9F=9F=E6=97=A5=E6=9C=9F=E6=B1=A1=E6=9F=93/=E9=83=A8?= =?UTF-8?q?=E5=88=86=E6=97=A5=E6=9C=9F=E8=8C=83=E5=9B=B4/=E6=A0=87?= =?UTF-8?q?=E8=AE=B0=E7=BB=84=E8=B7=AF=E5=BE=84=E7=AD=897=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/services/pubmed_query_parser.py | 141 +++++++++++--------- backend/app/services/search_engine.py | 100 ++++++++++++-- docs/13-搜索修复全记录.md | 81 +++++++++++ 3 files changed, 249 insertions(+), 73 deletions(-) diff --git a/backend/app/services/pubmed_query_parser.py b/backend/app/services/pubmed_query_parser.py index 986e20f..d117213 100644 --- a/backend/app/services/pubmed_query_parser.py +++ b/backend/app/services/pubmed_query_parser.py @@ -464,6 +464,13 @@ class PubmedQueryParser: if t.field in _DATE_RANGE_FIELDS: result._top_level_date_fields.add(t.field) + # R27: also track top-level date range markers (excluded from _ungrouped by _is_range_end filter) + for t in terms: + if getattr(t, '_is_range_end', False) and t.group_id < 0: + _tag = t.field.replace("__RANGE_", "").replace("__", "") + if _tag in _DATE_RANGE_FIELDS: + result._top_level_date_fields.add(_tag) + # P2-1: Handle unconsumed tokens (e.g., orphan text after RPAREN) if self.pos < len(self.tokens) - 1: for t in self.tokens[self.pos:-1]: # exclude EOF token @@ -935,8 +942,8 @@ class PubmedQueryParser: return [] token = self.advance() text = token.value.strip('"') if token.type == TokenType.QUOTED else token.value - # R21: skip empty quoted text ""[TI] - if not text: + # R21: skip empty quoted text ""[TI]; R27: also skip whitespace-only + if not text or not text.strip(): return [] is_exact = (token.type == TokenType.QUOTED) field = None @@ -1013,6 +1020,11 @@ class PubmedQueryParser: except IndexError: pass # P12: validate date values — non-numeric garbage falls back to plain text + # R27: expand partial dates YYYY-MM in ranges before validation + if _PARTIAL_DATE_RE.match(start_val): + start_val, _ = _expand_partial_date(start_val) + if _PARTIAL_DATE_RE.match(end_val): + _, end_val = _expand_partial_date(end_val) _valid_date = lambda s: _validate_date_str(s) if not _valid_date(start_val) or not _valid_date(end_val): txt = f"{start_val}:{end_val}[{field}]" @@ -1021,71 +1033,74 @@ class PubmedQueryParser: _start_is_year = start_val.isdigit() and len(start_val) == 4 _end_is_year = end_val.isdigit() and len(end_val) == 4 # Year-only range (e.g., 2024:2026[EDAT]) - if _start_is_year and _end_is_year: - if negated: - # R26: store in _neg_single_dates instead of main fields - result._neg_single_dates.setdefault(field, []).append( - (f"{start_val}-01-01", f"{end_val}-12-31") - ) - else: - try: - if yr_from_attr: - curr_f = getattr(result, yr_from_attr) - new_f = int(start_val) - setattr(result, yr_from_attr, max(curr_f, new_f) if curr_f is not None else new_f) - curr_t = getattr(result, yr_to_attr) - new_t = int(end_val) - setattr(result, yr_to_attr, min(curr_t, new_t) if curr_t is not None else new_t) - else: - # For non-DP date fields: convert year to full date for consistency - curr_f = getattr(result, date_attr) - new_f = f"{start_val}-01-01" - setattr(result, date_attr, max(curr_f, new_f) if curr_f is not None else new_f) - curr_t = getattr(result, date_attr_to) - new_t = f"{end_val}-12-31" - setattr(result, date_attr_to, min(curr_t, new_t) if curr_t is not None else new_t) - except (ValueError, TypeError): - pass - elif _start_is_year and not _end_is_year: - # Mixed: start is year, end is full date (e.g., 2024:2024-12-01[EDAT]) - if negated: - result._neg_single_dates.setdefault(field, []).append( - (f"{start_val}-01-01", end_val) - ) - else: - if yr_from_attr: + # R27: only set global attributes for top-level ranges (depth == 0) + # Ranges inside groups are handled via markers in the engine's group path. + if self._depth == 0: + if _start_is_year and _end_is_year: + if negated: + # R26: store in _neg_single_dates instead of main fields + result._neg_single_dates.setdefault(field, []).append( + (f"{start_val}-01-01", f"{end_val}-12-31") + ) + else: try: - curr = getattr(result, yr_from_attr) - v = int(start_val) - setattr(result, yr_from_attr, max(curr, v) if curr is not None else v) + if yr_from_attr: + curr_f = getattr(result, yr_from_attr) + new_f = int(start_val) + setattr(result, yr_from_attr, max(curr_f, new_f) if curr_f is not None else new_f) + curr_t = getattr(result, yr_to_attr) + new_t = int(end_val) + setattr(result, yr_to_attr, min(curr_t, new_t) if curr_t is not None else new_t) + else: + # For non-DP date fields: convert year to full date for consistency + curr_f = getattr(result, date_attr) + new_f = f"{start_val}-01-01" + setattr(result, date_attr, max(curr_f, new_f) if curr_f is not None else new_f) + curr_t = getattr(result, date_attr_to) + new_t = f"{end_val}-12-31" + setattr(result, date_attr_to, min(curr_t, new_t) if curr_t is not None else new_t) except (ValueError, TypeError): pass - curr_f = getattr(result, date_attr) - new_f = f"{start_val}-01-01" - setattr(result, date_attr, max(curr_f, new_f) if curr_f is not None else new_f) - curr_t = getattr(result, date_attr_to) - setattr(result, date_attr_to, min(curr_t, end_val) if curr_t is not None else end_val) - elif not _start_is_year and _end_is_year: - # Mixed: start is full date, end is year (e.g., 2024-01-01:2026[EDAT]) - if negated: - result._neg_single_dates.setdefault(field, []).append( - (start_val, f"{end_val}-12-31") - ) + elif _start_is_year and not _end_is_year: + # Mixed: start is year, end is full date (e.g., 2024:2024-12-01[EDAT]) + if negated: + result._neg_single_dates.setdefault(field, []).append( + (f"{start_val}-01-01", end_val) + ) + else: + if yr_from_attr: + try: + curr = getattr(result, yr_from_attr) + v = int(start_val) + setattr(result, yr_from_attr, max(curr, v) if curr is not None else v) + except (ValueError, TypeError): + pass + curr_f = getattr(result, date_attr) + new_f = f"{start_val}-01-01" + setattr(result, date_attr, max(curr_f, new_f) if curr_f is not None else new_f) + curr_t = getattr(result, date_attr_to) + setattr(result, date_attr_to, min(curr_t, end_val) if curr_t is not None else end_val) + elif not _start_is_year and _end_is_year: + # Mixed: start is full date, end is year (e.g., 2024-01-01:2026[EDAT]) + if negated: + result._neg_single_dates.setdefault(field, []).append( + (start_val, f"{end_val}-12-31") + ) + else: + curr_f = getattr(result, date_attr) + setattr(result, date_attr, max(curr_f, start_val) if curr_f is not None else start_val) + curr_t = getattr(result, date_attr_to) + v = f"{end_val}-12-31" + setattr(result, date_attr_to, min(curr_t, v) if curr_t is not None else v) else: - curr_f = getattr(result, date_attr) - setattr(result, date_attr, max(curr_f, start_val) if curr_f is not None else start_val) - curr_t = getattr(result, date_attr_to) - v = f"{end_val}-12-31" - setattr(result, date_attr_to, min(curr_t, v) if curr_t is not None else v) - else: - # Full date range (e.g., 2024-01-01:2024-12-31[EDAT]) - if negated: - result._neg_single_dates.setdefault(field, []).append((start_val, end_val)) - else: - curr_f = getattr(result, date_attr) - setattr(result, date_attr, max(curr_f, start_val) if curr_f is not None else start_val) - curr_t = getattr(result, date_attr_to) - setattr(result, date_attr_to, min(curr_t, end_val) if curr_t is not None else end_val) + # Full date range (e.g., 2024-01-01:2024-12-31[EDAT]) + if negated: + result._neg_single_dates.setdefault(field, []).append((start_val, end_val)) + else: + curr_f = getattr(result, date_attr) + setattr(result, date_attr, max(curr_f, start_val) if curr_f is not None else start_val) + curr_t = getattr(result, date_attr_to) + setattr(result, date_attr_to, min(curr_t, end_val) if curr_t is not None else end_val) marker = Term(f"{start_val}:{end_val}", field=marker_field, is_not=False) marker._is_range_end = True result._date_range_markers.append(marker) diff --git a/backend/app/services/search_engine.py b/backend/app/services/search_engine.py index 2f1cc56..38a3a97 100644 --- a/backend/app/services/search_engine.py +++ b/backend/app/services/search_engine.py @@ -1248,15 +1248,25 @@ class AdvancedSearchEngine: # R23-3: track date fields referenced in negated groups (De Morgan fix) _neg_group_date_fields: set[str] = set() for t in group: - # P15: __RANGE_* markers are side-effect-only + # R27: process date markers for ALL groups (follows text term pattern) if getattr(t, '_is_range_end', False): - if _negated: + _marker_cond = AdvancedSearchEngine._build_marker_condition(t) + if _negated and _marker_cond is not None: # R23-3: identify which date field this marker references _mf = t.field or "" if _mf.startswith("__RANGE_") and _mf.endswith("__"): _tag = _mf[8:-2] if _tag in ("DP", "EDAT", "CRDT", "MHDA", "LR", "DCOM", "DEP"): _neg_group_date_fields.add(_tag) + # R27: negated group → all go to g_neg + # is_not=True means inner NOT: not_(cond), outer NOT wraps: not_(not_(cond)) = cond + g_neg.append(not_(_marker_cond) if getattr(t, 'is_not', False) else _marker_cond) + elif _marker_cond is not None: + # Non-negated group: is_not → negated range; otherwise → positive + if getattr(t, 'is_not', False): + g_neg.append(not_(_marker_cond)) + else: + g_pos.append(_marker_cond) continue cond = await AdvancedSearchEngine._single_term_condition(db, t) if cond is not None: @@ -1297,12 +1307,9 @@ class AdvancedSearchEngine: else: g_pos.append(child_combined) - # R23-3: include date conditions inside negated group scope for correct De Morgan + # R27: date conditions are now built directly from markers in the group loop above. + # Only track handled fields for top-level skip logic. if _negated and _neg_group_date_fields: - for _tag in _neg_group_date_fields: - _date_cond = AdvancedSearchEngine._build_date_cond_from_pp(pp, _tag) - if _date_cond is not None: - g_neg.append(_date_cond) _handled_neg_group_date_fields.update(_neg_group_date_fields) gop = (pp.group_operators[idx] @@ -1342,11 +1349,15 @@ class AdvancedSearchEngine: if isinstance(c, UnaryExpression) and c.modifier == _sa_ops.inv: return True # R26: NULL-safe NOT: or_(not_(inner), col.is_(None)) + # R27: verify second clause is indeed is_(None) to avoid false positives try: if hasattr(c, 'operator') and c.operator is _sa_ops.or_: clauses = list(getattr(c, 'clauses', ())) if len(clauses) >= 2 and isinstance(clauses[0], UnaryExpression) and clauses[0].modifier == _sa_ops.inv: - return True + # Verify remaining clause is an IS NULL check + for _cl in clauses[1:]: + if hasattr(_cl, 'operator') and _cl.operator in (_sa_ops.is_, _sa_ops.isnot): + return True except Exception: pass return False @@ -1406,7 +1417,9 @@ class AdvancedSearchEngine: except ValueError: pass if neg_conds: - conditions.append(not_(and_(*neg_conds) if len(neg_conds) > 1 else neg_conds[0])) + # R27: NULL-safe NOT — rows with NULL pub_date should not be excluded + _range_cond = and_(*neg_conds) if len(neg_conds) > 1 else neg_conds[0] + conditions.append(or_(not_(_range_cond), GlobalLiterature.pub_date.is_(None))) elif dp_conds: cond = and_(*dp_conds) if len(dp_conds) > 1 else dp_conds[0] conditions.append(not_(cond) if dp_negated else cond) @@ -1462,7 +1475,9 @@ class AdvancedSearchEngine: except ValueError: pass if neg_conds: - conditions.append(not_(and_(*neg_conds) if len(neg_conds) > 1 else neg_conds[0])) + # R27: NULL-safe NOT — rows with NULL column should not be excluded + _range_cond = and_(*neg_conds) if len(neg_conds) > 1 else neg_conds[0] + conditions.append(or_(not_(_range_cond), col.is_(None))) elif field_conds: cond = and_(*field_conds) if len(field_conds) > 1 else field_conds[0] negated = field_tag in getattr(pp, 'negated_date_ranges', set()) @@ -2054,3 +2069,68 @@ class AdvancedSearchEngine: except ValueError: pass return and_(*conds) if conds else None + + @staticmethod + def _build_marker_condition(marker_term) -> object | None: + """Build SQL condition from a __RANGE_* marker term for group-internal date ranges. + + R27: used by the group processing path to build date conditions from markers, + replacing the old approach that read conflated global attributes. + """ + _mf = marker_term.field or "" + if not _mf.startswith("__RANGE_") or not _mf.endswith("__"): + return None + _tag = _mf[8:-2] + if _tag not in ("DP", "EDAT", "CRDT", "MHDA", "LR", "DCOM", "DEP"): + return None + + parts = marker_term.text.split(":", 1) + if len(parts) != 2: + return None + _from_str, _to_str = parts + + from datetime import date as _dt_date + + if _tag == "DP": + conds = [] + if _from_str.isdigit() and len(_from_str) == 4: + conds.append(GlobalLiterature.pub_year >= int(_from_str)) + else: + try: + conds.append(GlobalLiterature.pub_date >= _dt_date.fromisoformat(_from_str)) + except ValueError: + pass + if _to_str.isdigit() and len(_to_str) == 4: + conds.append(GlobalLiterature.pub_year <= int(_to_str)) + else: + try: + conds.append(GlobalLiterature.pub_date <= _dt_date.fromisoformat(_to_str)) + except ValueError: + pass + return and_(*conds) if conds else None + + _DATE_COL_MAP = { + "EDAT": GlobalLiterature.entrez_date, + "CRDT": GlobalLiterature.create_date, + "MHDA": GlobalLiterature.meshed_date, + "LR": GlobalLiterature.pubmed_revised, + "DCOM": GlobalLiterature.date_completed, + "DEP": GlobalLiterature.pub_date, + } + col = _DATE_COL_MAP.get(_tag) + if col is None: + return None + conds = [] + if _from_str.isdigit() and len(_from_str) == 4: + _from_str = f"{_from_str}-01-01" + try: + conds.append(col >= _dt_date.fromisoformat(_from_str)) + except ValueError: + pass + if _to_str.isdigit() and len(_to_str) == 4: + _to_str = f"{_to_str}-12-31" + try: + conds.append(col <= _dt_date.fromisoformat(_to_str)) + except ValueError: + pass + return and_(*conds) if conds else None diff --git a/docs/13-搜索修复全记录.md b/docs/13-搜索修复全记录.md index 377f861..0a7f11d 100644 --- a/docs/13-搜索修复全记录.md +++ b/docs/13-搜索修复全记录.md @@ -2014,3 +2014,84 @@ def _is_negated_cond(c): | Bug 3 | LOW | Facet cache 从未写入 p2+（`total=0` 回退） | | Bug 4 | LOW | 无效日期字段标签静默降级为全字段搜索 | | Bug 5 | LOW | 否定组内冗余日期条件（单个年份 + 全范围） | + +--- + +# 第27轮审计修复 (R27) + +## 背景 + +第 27 轮由 3 个并行审计 agent 覆盖：解析器、引擎、前端/集成。发现 2 个 CRITICAL 引擎-解析器交互 bug、1 个 MEDIUM 解析器 bug，以及若干 LOW 问题。 + +## 修复清单 + +### R27-1 (CRITICAL): 日期范围属性跨作用域污染 + +**文件**：[pubmed_query_parser.py:1024-1088](backend/app/services/pubmed_query_parser.py#L1024) + [search_engine.py:1300-1306](backend/app/services/search_engine.py#L1300) + +**根因**：`_parse_range` 对所有非否定日期范围都用 `max/min` intersect 更新全局 `edat_from/edat_to` 属性。当同一日期字段同时出现在顶层和括号组内（如 `2000:2010[EDAT] NOT (cancer AND 2005:2006[EDAT])`），顶层范围被组内范围错误缩小：`edat_from` 从 `2000-01-01` 变成 `2005-01-01`，`edat_to` 从 `2010-12-31` 变成 `2006-12-31`。加上 `_top_level_date_fields` 未追踪 `__RANGE_*` 标记，导致 `_neg_only` 判定跳过整个日期节。 + +**修复**： +- `_parse_range` 增加 `if self._depth == 0` 守卫——仅顶层范围更新全局属性，组内范围不再污染 +- `parse()` 循环 `terms` 追踪 `__RANGE_*` 标记的 `_top_level_date_fields` +- 新增 `_build_marker_condition()` 静态方法——从标记内嵌的 `start_val:end_val` 文本重建 SQL 条件 +- 组循环中处理标记：非否定组 → `g_pos.append`，否定组 → `g_neg.append` +- 移除 `_build_date_cond_from_pp` 在否定组中的调用（已被标记直接处理取代） + +### R27-2 (MEDIUM): 范围语法中的部分日期 YYYY-MM 静默降级 + +**文件**：[pubmed_query_parser.py:1015-1017](backend/app/services/pubmed_query_parser.py#L1015) + +**根因**：`_validate_date_str` 只接受 `YYYY` 或 `YYYY-MM-DD`。`2024-01:2024-06[DP]` 中两个端点无法通过验证，整个范围降级为纯文本。单值部分日期（`2024-01[DP]`）因 R26 的 `_expand_partial_date` 调用路径已正确处理，但 `_parse_range` 缺少相同调用。 + +**修复**：在 `_parse_range` 的验证前添加 `_PARTIAL_DATE_RE` 匹配 + `_expand_partial_date` 展开： + +```python +if _PARTIAL_DATE_RE.match(start_val): + start_val, _ = _expand_partial_date(start_val) +if _PARTIAL_DATE_RE.match(end_val): + _, end_val = _expand_partial_date(end_val) +``` + +### R27-3 (MEDIUM): `_is_negated_cond` NULL-safe 模式验证不完整 + +**文件**：[search_engine.py:1345-1351](backend/app/services/search_engine.py#L1345) + +**根因**：NULL-safe NOT 检测只检查 `or_` 的第一个子句是 `not_(...)`，没有验证第二个子句是 `is_(None)`。潜在误报：`or_(not_(X), Y)` 会被错误识别为否定。 + +**修复**：增加对第二子句的 `is_(None)` / `isnot(None)` 验证。 + +### R27-4 (LOW): `_neg_single_dates` NOT 条件缺少 NULL 安全 + +**文件**：[search_engine.py:1421, 1477](backend/app/services/search_engine.py#L1421) + +**根因**：`not_(and_(col >= from, col <= to))` — 当 `col` 为 NULL 时，`NULL >= date` → NULL，`NOT(NULL)` → NULL（WHERE 中为假）。NULL 日期行的记录被错误排除。 + +**修复**：使用 `or_(not_(_range_cond), col.is_(None))`。 + +### R27-5 (LOW): 空白字符引号文本不跳过 + +**文件**：[pubmed_query_parser.py:939](backend/app/services/pubmed_query_parser.py#L939) + +**根因**：`" "[TI]` — `text = " "`（两个空格），`if not text` 为 False，创建搜索条件包含双空格。 + +**修复**：改为 `if not text or not text.strip(): return []`。 + +## 前端审计发现（R27 未修复） + +| 编号 | 严重度 | 描述 | 文件 | +|------|--------|------|------| +| C-1 | CRITICAL | 非 keyset 排序（best_match/relevance）第 2 页起总分页消失 | SearchView.vue:278 | +| C-2 | CRITICAL | `#N` 引用在 SearchView 中不解析 | SearchView.vue:865 | +| H-1 | HIGH | offset 分页第 2 页起 total 显示为 0 | SearchView.vue:278,353 | +| H-2 | HIGH | 错误时 goToPage 不回退页码 | SearchView.vue:369-380 | +| M-1 | MEDIUM | 空查询时年份直方图与"未找到"同时显示 | SearchView.vue:625 | +| M-3 | MEDIUM | URL 同时存储 `date_preset` 和绝对日期 | SearchView.vue:512-521 | +| L-3 | LOW | 空查询无筛选时返回全库年份分布 | search_engine.py:622-641 | + +## 验证 + +- ✅ **110 search tests passed**（parser + search engine + integration） +- 解析器 36 测试全部通过 +- 5 个新增 R27 正确性检查通过 +- 全量套件中仅外部服务连接失败（httpx.ConnectError），与改动无关