From 475d6bb80cf69885539787d72e0019bf5e669757 Mon Sep 17 00:00:00 2001 From: "34047007@qq.com" <34047007@qq.com> Date: Wed, 29 Jul 2026 06:16:11 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E7=AC=AC29=E8=BD=AE=E6=90=9C=E7=B4=A2?= =?UTF-8?q?=E5=AE=A1=E8=AE=A1=E4=BF=AE=E5=A4=8D=20=E2=80=94=20=E7=BC=BA?= =?UTF-8?q?=E5=A4=B1=E5=AF=BC=E5=85=A5/=E4=B8=AD=E6=96=87=E7=B2=BE?= =?UTF-8?q?=E7=A1=AE=E7=9F=AD=E8=AF=ADILIKE/=E5=AD=90=E7=BB=84=E6=97=A5?= =?UTF-8?q?=E6=9C=9F=E6=A0=87=E8=AE=B0/=E5=BC=80=E5=8C=BA=E9=97=B4?= =?UTF-8?q?=E8=8C=83=E5=9B=B4/NULL=E5=AE=89=E5=85=A8=E7=AD=898=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/services/pubmed_query_parser.py | 108 +++++++++++++++++++- backend/app/services/search_engine.py | 51 ++++++++- docs/13-搜索修复全记录.md | 82 +++++++++++++++ 3 files changed, 234 insertions(+), 7 deletions(-) diff --git a/backend/app/services/pubmed_query_parser.py b/backend/app/services/pubmed_query_parser.py index 56c53f5..bdc577f 100644 --- a/backend/app/services/pubmed_query_parser.py +++ b/backend/app/services/pubmed_query_parser.py @@ -932,6 +932,19 @@ class PubmedQueryParser: t1 = self.peek_n(1) t2 = self.peek_n(2) + # R29: open-ended start range :2024[DP] → COLON NUMBER/DATE FIELD + if t0.type == TokenType.COLON and t1 is not None and t1.type in (TokenType.NUMBER, TokenType.DATE): + self.advance() # consume COLON + e_val = self.advance().value # end value + return self._handle_date_range_edge(result, negated, start_val=None, end_val=e_val) + + # R29: open-ended end range 2024:[DP] → NUMBER/DATE COLON FIELD + if (t0.type in (TokenType.NUMBER, TokenType.DATE) and t1 is not None and t1.type == TokenType.COLON + and t2 is not None and t2.type == TokenType.FIELD): + s_val = self.advance().value # start value + self.advance() # consume COLON + return self._handle_date_range_edge(result, negated, start_val=s_val, end_val=None) + if (t1 is not None and t1.type == TokenType.COLON and t2 is not None and t0.type in (TokenType.NUMBER, TokenType.DATE, TokenType.WORD) @@ -1110,9 +1123,96 @@ class PubmedQueryParser: # Non-date range or no field → plain text txt = f"{start_val}:{end_val}" - if field: - txt = f"{txt}[{field}]" - return [Term(txt, field=field, is_not=False)] + # R29: field tag is already stored in Term.field — do not append to text + return [Term(txt, field=field or "", is_not=False)] + + # ─── R29: Open-ended date range helper ─── + def _handle_date_range_edge(self, result: ParsedPubmedQuery, negated: bool = False, + start_val: str | None = None, end_val: str | None = None) -> list[Term]: + """Handle open-ended ranges like :2024[DP] or 2024:[DP].""" + field = None + if self.peek().type == TokenType.FIELD: + ft = self.advance() + field = ft.value[1:-1].upper() + _norm = _normalize_field_label(field) + if _norm is not None: + field = _norm + + if field not in _DATE_RANGE_FIELDS: + txt = f"{start_val or ''}:{end_val or ''}" + return [Term(txt, field=field or "", is_not=False)] + + attr_map = { + "DP": ("date_from", "date_to", "year_from", "year_to", "__RANGE_DP__"), + "EDAT": ("edat_from", "edat_to", None, None, "__RANGE_EDAT__"), + "CRDT": ("crdt_from", "crdt_to", None, None, "__RANGE_CRDT__"), + "MHDA": ("mhda_from", "mhda_to", None, None, "__RANGE_MHDA__"), + "LR": ("lr_from", "lr_to", None, None, "__RANGE_LR__"), + "DCOM": ("dcom_from", "dcom_to", None, None, "__RANGE_DCOM__"), + "DEP": ("dep_from", "dep_to", None, None, "__RANGE_DEP__"), + } + date_attr, date_attr_to, yr_from_attr, yr_to_attr, marker_field = attr_map[field] + + if self._depth == 0: + if start_val is not None and end_val is None: + # Open-ended end: 2024:[DP] → from start_val onwards + if start_val.isdigit() and len(start_val) == 4: + if yr_from_attr is not None: + if negated: + result._neg_single_dates.setdefault(field, []).append( + (f"{start_val}-01-01", None)) + else: + curr = getattr(result, yr_from_attr) + v = int(start_val) + setattr(result, yr_from_attr, max(curr, v) if curr is not None else v) + else: + if negated: + result._neg_single_dates.setdefault(field, []).append( + (f"{start_val}-01-01", None)) + else: + curr = getattr(result, date_attr) + v = f"{start_val}-01-01" + setattr(result, date_attr, max(curr, v) if curr is not None else v) + else: + if _PARTIAL_DATE_RE.match(start_val): + start_val, _ = _expand_partial_date(start_val) + if negated: + result._neg_single_dates.setdefault(field, []).append((start_val, None)) + else: + curr = getattr(result, date_attr) + setattr(result, date_attr, max(curr, start_val) if curr is not None else start_val) + elif end_val is not None and start_val is None: + # Open-ended start: :2024[DP] → up to end_val + if end_val.isdigit() and len(end_val) == 4: + if yr_to_attr is not None: + if negated: + result._neg_single_dates.setdefault(field, []).append( + (None, f"{end_val}-12-31")) + else: + curr = getattr(result, yr_to_attr) + v = int(end_val) + setattr(result, yr_to_attr, min(curr, v) if curr is not None else v) + else: + if negated: + result._neg_single_dates.setdefault(field, []).append( + (None, f"{end_val}-12-31")) + else: + curr = getattr(result, date_attr_to) + v = f"{end_val}-12-31" + setattr(result, date_attr_to, min(curr, v) if curr is not None else v) + else: + if _PARTIAL_DATE_RE.match(end_val): + _, end_val = _expand_partial_date(end_val) + if negated: + result._neg_single_dates.setdefault(field, []).append((None, end_val)) + else: + curr = getattr(result, date_attr_to) + setattr(result, date_attr_to, min(curr, end_val) if curr is not None else end_val) + + marker = Term(f"{start_val or ''}:{end_val or ''}", field=marker_field, is_not=False) + marker._is_range_end = True + result._date_range_markers.append(marker) + return [marker] # ─── Public API ─── @@ -1198,7 +1298,7 @@ def extract_pubmed_query_for_prisma(query: str) -> tuple[str, list[str]]: # P5: Handle field tags with '/' (e.g. Title/Article) or special chars normalized = re.sub( - r'\[([\w/:]+)\]', + r'\[([\w/: -]+)\]', lambda m: f'[{m.group(1).upper()}]', query, ) diff --git a/backend/app/services/search_engine.py b/backend/app/services/search_engine.py index 1d664ba..a0227c1 100644 --- a/backend/app/services/search_engine.py +++ b/backend/app/services/search_engine.py @@ -560,7 +560,7 @@ class AdvancedSearchEngine: GlobalLiterature.abstract.isnot(None), GlobalLiterature.abstract != '', )) - if is_free_full_text and is_oa is None: + if is_free_full_text: conditions.append(GlobalLiterature.is_oa == True) if has_full_text: conditions.append(GlobalLiterature.pmc_id.isnot(None)) @@ -865,6 +865,19 @@ class AdvancedSearchEngine: "pages": pp.pages_terms, "lid": pp.lid_terms, } + _NULL_SAFE_COL_MAP = { + "title": GlobalLiterature.title, + "abstract": GlobalLiterature.abstract, + "author": GlobalLiterature.author_names_text, + "journal": GlobalLiterature.journal, + "affiliation": GlobalLiterature.authors, + "language": GlobalLiterature.language, + "volume": GlobalLiterature.volume, + "issue": GlobalLiterature.issue, + "pages": GlobalLiterature.pages, + "lid": GlobalLiterature.doi, + "all": GlobalLiterature.title, + } for fld, terms in field_map.items(): if not terms: continue @@ -873,7 +886,11 @@ class AdvancedSearchEngine: for term in terms: cond = AdvancedSearchEngine._field_condition(fld, term.text, term.exact) if term.is_not: - neg_conds.append(not_(cond)) + _ns_col = _NULL_SAFE_COL_MAP.get(fld) + if _ns_col is not None: + neg_conds.append(or_(not_(cond), _ns_col.is_(None))) + else: + neg_conds.append(not_(cond)) else: pos_conds.append(cond) if pos_conds: @@ -1311,6 +1328,31 @@ class AdvancedSearchEngine: child_neg = [] for t in child_group: if getattr(t, '_is_range_end', False): + # R29: process date range markers in child sub-groups + _c_marker_cond = AdvancedSearchEngine._build_marker_condition(t) + if _c_marker_cond is not None: + if getattr(t, 'is_not', False): + _cmf = t.field or "" + _ctag = _cmf[8:-2] if _cmf.startswith("__RANGE_") and _cmf.endswith("__") else None + if _ctag == "DP": + child_neg.append(or_(not_(_c_marker_cond), GlobalLiterature.pub_year.is_(None), GlobalLiterature.pub_date.is_(None))) + elif _ctag in ("EDAT", "CRDT", "MHDA", "LR", "DCOM", "DEP"): + _ctag_col = { + "EDAT": GlobalLiterature.entrez_date, + "CRDT": GlobalLiterature.create_date, + "MHDA": GlobalLiterature.meshed_date, + "LR": GlobalLiterature.pubmed_revised, + "DCOM": GlobalLiterature.date_completed, + "DEP": GlobalLiterature.pub_date, + }.get(_ctag) + if _ctag_col: + child_neg.append(or_(not_(_c_marker_cond), _ctag_col.is_(None))) + else: + child_neg.append(not_(_c_marker_cond)) + else: + child_neg.append(not_(_c_marker_cond)) + else: + child_pos.append(_c_marker_cond) continue cond = await AdvancedSearchEngine._single_term_condition(db, t) if cond is not None: @@ -1691,7 +1733,7 @@ class AdvancedSearchEngine: from datetime import date as _d return and_(col >= _d(year, 1, 1), col <= _d(year, 12, 31)) # R28: partial date YYYY-MM inside group - from app.services.pubmed_query_parser import _PARTIAL_DATE_RE as _PDR + from app.services.pubmed_query_parser import _PARTIAL_DATE_RE as _PDR, _expand_partial_date if _PDR.match(_term_text): _df, _dt = _expand_partial_date(_term_text) from datetime import date as _d @@ -1789,8 +1831,11 @@ class AdvancedSearchEngine: if exact and not _wildcard: # P4: 精确短语 → phraseto_tsquery（利用 GIN 索引，保留词序） # P13: 加入 journal/journal_iso ILIKE（不在 tsvector 中） + # R29: 加入 title/abstract ILIKE（非英文如中文无法被 tsquery 匹配） return or_( GlobalLiterature.search_tsv.op("@@")(func.phraseto_tsquery("english", term)), + GlobalLiterature.title.ilike(pat), + GlobalLiterature.abstract.ilike(pat), GlobalLiterature.journal.ilike(pat), GlobalLiterature.journal_iso.ilike(pat), text("EXISTS (SELECT 1 FROM jsonb_array_elements(global_literature.authors) AS _e " diff --git a/docs/13-搜索修复全记录.md b/docs/13-搜索修复全记录.md index edd1a8f..d55c7f5 100644 --- a/docs/13-搜索修复全记录.md +++ b/docs/13-搜索修复全记录.md @@ -2155,3 +2155,85 @@ if _PARTIAL_DATE_RE.match(end_val): - ✅ **1007 tests passed, 0 failed** - 全部搜索测试通过（parser 36 + search engine 集成测试） - 无回归 + +## R29 (2026-07-29): 第四轮并行审计修复 + +### 审计范围 + +4 路并行 agent，分别覆盖： +1. `pubmed_query_parser.py` 语法正确性（递归下降解析器） +2. `search_engine.py` 条件构建（_pubmed_conditions 全字段路径） +3. 纯文本降级路径（_is_flat_text 决策、缓存键、引号短语、facet 一致性） +4. 前端搜索参数映射（前后端参数完整性、URL 同步、分页） + +### R29-1 (P0): `_expand_partial_date` 未导入 + +**文件**：[search_engine.py:1696](backend/app/services/search_engine.py#L1696) + +**根因**：`_single_term_condition()` 的 YYYY-MM 部分日期分支调用 `_expand_partial_date()` 但该函数未从 `pubmed_query_parser` 导入。导入语句只导入了 `_PARTIAL_DATE_RE` 和 `_validate_date_str`。 + +**影响**：组内 YYYY-MM 日期（如 `(2024-06[DP] OR cancer)`）触发 `NameError`，被 `except Exception` 吞没后降级到纯文本。 + +**修复**：添加 `_expand_partial_date` 到 inline import。 + +### R29-2 (P0): 非日期范围 field 标签混入文本 + +**文件**：[pubmed_query_parser.py:1111-1115](backend/app/services/pubmed_query_parser.py#L1111) + +**根因**：`_parse_range()` 对非日期范围（如 `term1:term2[MH]`）的 fallthrough 路径将 `[MH]` 追加入 term 文本，导致 `text="term1:term2[MH]"` 而 `Term.field="MH"` 已正确设置。引擎按 `field` 路由（正确到 MeSH 路径），但 `text` 中的 `[MH]` 被作为字面搜索词。 + +**修复**：去除 `[{field}]` 文本拼接，只保留 `start_val:end_val`。 + +### R29-3 (P0): 中文精确短语缺少 title/abstract ILIKE + +**文件**：[search_engine.py:1789-1798](backend/app/services/search_engine.py#L1789) + +**根因**：`_field_condition("all", exact=True)` 分支使用 `phraseto_tsquery('english', ...)` 无法匹配非英文文本（如中文）。缺少 title/abstract ILIKE 回退。非精确分支（line 1832）正确包含这些 ILIKE。 + +**影响**：`"肺癌"[TIAB]` 等中文精确短语返回零结果。 + +**修复**：在 exact 分支中添加 `title.ilike(pat)` 和 `abstract.ilike(pat)`。 + +### R29-4 (P1): 子组日期范围标记被丢弃 + +**文件**：[search_engine.py:1313-1314](backend/app/services/search_engine.py#L1313) + +**根因**：子组（sub-group，由 `_parse_or_expr` AND 集群 OR 拆分产生）的日期范围标记在 child processing 循环中被 `continue` 跳过。 + +**影响**：`(A OR B AND 2020:2024[DP])` 等查询中日期范围丢失。 + +**修复**：子组循环中添加 `_build_marker_condition` + NULL 安全 NOT，逻辑与父组一致。 + +### R29-5 (P1): 开区间日期范围不支持 + +**文件**：[pubmed_query_parser.py:930-938](backend/app/services/pubmed_query_parser.py#L930) + +**根因**：`:2024[DP]`（截止到 2024）和 `2024:[DP]`（从 2024 起）的 COLON 在首个位置或无第二个 NUMBER 的模式不被范围检测匹配。 + +**修复**：在 `_parse_atom` 中增加 `:NUMBER[FIELD]` 和 `NUMBER:[FIELD]` 检测，路由到新方法 `_handle_date_range_edge()`，正确设置全局属性、标记和 `_neg_single_dates`。 + +### R29-6 (P2): field_map NOT 条件缺少 NULL 安全 + +**文件**：[search_engine.py:874-881](backend/app/services/search_engine.py#L874) + +**根因**：`[TI]`/`[AB]`/`[AU]` 等字段标签的否定词使用裸 `not_(cond)`，导致字段为 NULL 的行被错误排除。 + +**修复**：添加 `_NULL_SAFE_COL_MAP`，对每个字段的否定条件使用 `or_(not_(cond), col.is_(None))`。 + +### R29-7 (LOW): PRISMA 导出正则不匹配含空格的字段标签 + +**文件**：[pubmed_query_parser.py:1301](backend/app/services/pubmed_query_parser.py#L1301) + +**修复**：`[\w/:]+` → `[\w/: -]+` 以匹配 `[MeSH Terms]`、`[Date - Publication]` 等。 + +### R29-8 (LOW): `is_free_full_text` 在 `is_oa` 已设时被跳过 + +**文件**：[search_engine.py:563](backend/app/services/search_engine.py#L563) + +**修复**：移除 `is_oa is None` 守卫，`is_free_full_text` 始终独立生效。 + +## 验证 + +- ✅ **1007 tests passed, 0 failed** +- 全部搜索测试通过 +- 无回归