Compare commits

..
2 Commits
Author SHA1 Message Date
34047007@qq.com 89e154caa2 docs: 第11轮搜索审计修复记录(16项P0-P2修复)
CI / backend (push) Canceled after 0s
CI / frontend (push) Canceled after 0s
2026-07-28 12:09:55 +08:00
34047007@qq.com 090938fa2b fix: 第11轮深度审计修复 — P0 keyset翻页崩溃 + P1布尔符语义/语法误报等16项修复
P0 (1项):
- keyset cursor: lit.title or "__NULL__" 空字符串误判为NULL导致后续翻页全空

P1 (6项):
- boolean_operator: 仅扫描depth=0的token,括号内AND/OR不再影响顶层操作符判定
- is_pubmed_syntax: 去除AND/OR/NOT检测的IGNORECASE(PubMed仅识别大写布尔符)
- _relevance_query: MeSH-only查询(如breast[MAJR])补充原始回退,避免退化到日期排序
- pmid_terms: 缺少try/except int(),补充ValueError+DOI兜底
- 部分日期展开: 2024-01[DP]等YYYY-MM partial date展开为整月范围(DP/EDAT/CRDT)
- 前端#N去重: resolveQuery/expandQuery dedup refs防重复引用误判为循环

P2 (9项):
- cache: filter-options加入invalidate_search_cache清理
- worker: daily_ftp_update/daily_citation_update异常时finally保证缓存清理
- cursor_val: 空字符串''通过is None检查,补充not cursor_val
- keyset UI: 模板条件sort==='date'改为KEYSET_SORTS.has(sort)
- resetAllFilters: 补充showCustomYear=false
2026-07-28 12:08:17 +08:00
11 changed files with 164 additions and 27 deletions
+1
View File
@@ -170,6 +170,7 @@ class CacheService:
await self.delete("search:year_counts:all")
await self.delete("journals:map")
await self.delete_pattern("atm:*")
await self.delete("filter-options")
async def invalidate_tenant(self, tenant_id: str):
await self.delete(f"tenant:{tenant_id}:plan")
+34 -4
View File
@@ -23,6 +23,19 @@ import unicodedata
from dataclasses import dataclass, field
from enum import Enum, auto
# P11: partial date YYYY-MM pattern, expanded to full month range
_PARTIAL_DATE_RE = re.compile(r'^\d{4}-\d{2}$')
_LAST_DAY = {1:31, 2:29, 3:31, 4:30, 5:31, 6:30, 7:31, 8:31, 9:30, 10:31, 11:30, 12:31}
def _expand_partial_date(text: str) -> tuple[str, str]:
"""Expand YYYY-MM to full month range (YYYY-MM-01 to YYYY-MM-last_day)."""
date_from = f"{text}-01"
y, m = text.split("-")
last_day = _LAST_DAY.get(int(m), 31)
date_to = f"{y}-{m}-{last_day}"
return date_from, date_to
# ─── 查询复杂度限制 ───
MAX_TERMS = 100 # P4-1: 放宽到 100 词(原 50 词)
MAX_PAREN_DEPTH = 10 # 括号嵌套最深层数
@@ -308,9 +321,20 @@ class PubmedQueryParser:
terms = self._parse_or_expr(result)
# 解析错误由 parse_pubmed_query 统一降级处理
# Detect boolean operator from token stream
has_and = any(t.type == TokenType.AND for t in self.tokens)
has_or = any(t.type == TokenType.OR for t in self.tokens)
# Detect boolean operator from top-level (depth=0) token stream only
# P11: 括号内的 AND/OR 不应影响顶层布尔操作符判定
depth = 0
has_and = has_or = False
for t in self.tokens:
if t.type == TokenType.LPAREN:
depth += 1
elif t.type == TokenType.RPAREN:
depth -= 1
elif depth == 0:
if t.type == TokenType.AND:
has_and = True
elif t.type == TokenType.OR:
has_or = True
if has_and and has_or:
result.boolean_operator = "mixed"
elif has_or and not has_and:
@@ -398,6 +422,8 @@ class PubmedQueryParser:
if term.text.isdigit() and len(term.text) == 4:
result.year_from = int(term.text)
result.year_to = int(term.text)
elif _PARTIAL_DATE_RE.match(term.text):
result.date_from, result.date_to = _expand_partial_date(term.text)
else:
result.date_from = term.text
result.date_to = term.text
@@ -407,6 +433,8 @@ class PubmedQueryParser:
if term.text.isdigit() and len(term.text) == 4:
result.year_from = int(term.text)
result.year_to = int(term.text)
elif _PARTIAL_DATE_RE.match(term.text):
result.edat_from, result.edat_to = _expand_partial_date(term.text)
else:
result.edat_from = term.text
result.edat_to = term.text
@@ -416,6 +444,8 @@ class PubmedQueryParser:
if term.text.isdigit() and len(term.text) == 4:
result.year_from = int(term.text)
result.year_to = int(term.text)
elif _PARTIAL_DATE_RE.match(term.text):
result.crdt_from, result.crdt_to = _expand_partial_date(term.text)
else:
result.crdt_from = term.text
result.crdt_to = term.text
@@ -749,7 +779,7 @@ def is_pubmed_syntax(query: str) -> bool:
query = unicodedata.normalize('NFKC', query)
if re.search(r'\[(' + '|'.join(_ALL_FIELD_TAGS) + r')\]', query, re.IGNORECASE):
return True
if re.search(r'\b(AND|OR|NOT)\b', query, re.IGNORECASE):
if re.search(r'\b(AND|OR|NOT)\b', query):
return True
return False
+10 -5
View File
@@ -609,7 +609,9 @@ class AdvancedSearchEngine:
plain_parts += [t.text for t in _pubmed_parsed.title_terms]
plain_parts += [t.text for t in _pubmed_parsed.abstract_terms]
plain_parts += [t.text for t in _pubmed_parsed.tiab_terms]
_relevance_query = " ".join(plain_parts) if plain_parts else ""
# P11: MeSH-only 查询(如 breast[MAJR])会产生空 plain_parts
# 退回到原始查询字符串保证相关性排序不退化到日期排序
_relevance_query = " ".join(plain_parts).strip() or query
# ── 通用 keyset 分页(所有列式排序模式统一,代替 OFFSET) ──
_keyset_cond = AdvancedSearchEngine._keyset_condition(sort, cursor_val, cursor_id)
q = select(GlobalLiterature)
@@ -1227,7 +1229,10 @@ class AdvancedSearchEngine:
# 7. [PMID] → 精确匹配,支持 is_not
for term in pp.pmid_terms:
cond = GlobalLiterature.pmid == int(term.text)
try:
cond = GlobalLiterature.pmid == int(term.text)
except ValueError:
cond = GlobalLiterature.doi.ilike(f"%{_escape_ilike(term.text)}%")
if term.is_not:
cond = not_(cond)
conditions.append(cond)
@@ -1621,7 +1626,7 @@ class AdvancedSearchEngine:
"""
if sort not in AdvancedSearchEngine.KEYSET_COLUMN_SORTS:
return None
if cursor_val is None or cursor_id is None:
if not cursor_val or cursor_id is None:
return None
import uuid as _uuid
try:
@@ -1690,9 +1695,9 @@ class AdvancedSearchEngine:
return str(lit.cited_by_count)
return "__NULL__"
elif sort == "title":
return lit.title or "__NULL__"
return lit.title # title is NOT NULL per model
elif sort == "journal":
return lit.journal or "__NULL__"
return lit.journal if lit.journal is not None else "__NULL__"
elif sort == "first_author":
authors = lit.authors or []
return authors[0].get("family") if authors else "__NULL__"
+12 -5
View File
@@ -24,11 +24,13 @@ async def shutdown(ctx):
async def daily_ftp_update(ctx):
"""每日 FTP 增量更新(取代旧的精搜+宽搜+retagger)"""
stats = await run_daily_ftp_update(ctx)
# 搜索缓存失效:管道新增/修改文献后,缓存中的搜索结果立即过时
from app.core.cache import cache
await cache.invalidate_search_cache()
return stats if isinstance(stats, dict) else {"status": "ok"}
try:
stats = await run_daily_ftp_update(ctx)
return stats if isinstance(stats, dict) else {"status": "ok"}
finally:
# 搜索缓存失效:无论管道成功还是异常,都保证清理
await cache.invalidate_search_cache()
async def daily_digest_task(ctx):
@@ -40,7 +42,12 @@ async def daily_digest_task(ctx):
async def daily_citation_update(ctx):
"""每日引用次数更新任务"""
from app.services.citation_updater import run_citation_update
return await run_citation_update(cached_hours=24)
from app.core.cache import cache
try:
stats = await run_citation_update(cached_hours=24)
return stats if isinstance(stats, dict) else {"status": "ok"}
finally:
await cache.invalidate_search_cache()
async def refresh_hot_articles_cache(ctx):
+3 -3
View File
@@ -192,13 +192,13 @@ class TestParserGaps:
"""((lung OR breast) AND therapy) — nested or flat groups"""
r = parse_pubmed_query("((lung OR breast) AND therapy)")
assert len(r.groups) >= 1
# outer paren: ((a OR b) AND therapy) — should have at least one group
assert r.boolean_operator == "mixed"
# P11: 括号内的 OR 不影响顶层操作符,顶层全部在括号内 → 无顶层 AND/OR → "and"
assert r.boolean_operator == "and"
def test_a4e_double_paren_operators(self):
"""((lung OR breast) AND therapy[TI]) — mixed ops"""
r = parse_pubmed_query("((lung OR breast) AND therapy[TI])")
assert r.boolean_operator == "mixed"
assert r.boolean_operator == "and" # P11: 括号内的 OR 不影响顶层
assert len(r.groups) >= 1
assert r.has_not is False
+4 -3
View File
@@ -32,8 +32,8 @@ class TestIsPubmedSyntax:
assert not is_pubmed_syntax(None)
def test_lowercase_boolean_detected(self):
"""小写的 and/or/not 识别为 PubMed 语法(P0-1 修复"""
assert is_pubmed_syntax("cancer and therapy")
"""小写的 and/or/not 识别为 PubMed 语法(P11: 仅大写 AND/OR/NOT 才是 PubMed 布尔符"""
assert not is_pubmed_syntax("cancer and therapy")
class TestTokenise:
@@ -152,7 +152,8 @@ class TestParsePubmedQuery:
# 括号分组后,term 进入 groups 而非直接列在 title_terms
assert len(result.groups) > 0
assert len(result.tiab_terms) == 1
assert result.boolean_operator == "mixed"
assert result.boolean_operator == "and"
# P11: 括号内的 OR 不影响顶层布尔操作符判定,顶层只有 AND
# 确保 groups 中包含括号内的两个 title 词
group_texts = [t.text for g in result.groups for t in g]
assert "lung cancer" in group_texts
@@ -172,7 +172,8 @@ class TestAllFrontendQueryFormats:
assert r.mesh_terms[0].text == "mouse"
assert r.mesh_terms[0].is_not is True
assert r.has_not is True
assert r.boolean_operator == "mixed"
assert r.boolean_operator == "and"
# P11: 括号内的 OR 不影响顶层,顶层只有 AND + NOT → boolean_operator="and"NOT 通过 has_not 处理
assert len(r.groups) == 1
group_texts = [t.text for t in r.groups[0]]
assert "lung cancer" in group_texts
+92 -3
View File
@@ -2,7 +2,7 @@
> 本文档按修复轮次详细记录所有搜索功能合规性修复的背景、根因分析和修改内容。
>
> **累计**10 轮,121 项修复,50+ 字段标签注册,1007 项测试覆盖,7 项已知限制
> **累计**11 轮,137 项修复,50+ 字段标签注册,1007 项测试覆盖,7 项已知限制
> **时间跨度**2026-07-24 ~ 2026-07-28
> **核心文件**`pubmed_query_parser.py`~730 行)→ `search_engine.py`~1350 行)
@@ -20,7 +20,8 @@
8. [第八轮:第 8 轮深度审计修复(12 项)](#第八轮第-8-轮深度审计修复)
9. [第九轮:第 9 轮深度审计修复(5 项)](#第九轮第-9-轮深度审计修复)
10. [第十轮:第 10 轮深度审计修复(6 项)](#第十轮第-10-轮深度审计修复)
11. [遗留限制](#遗留限制)
11. [第十一轮:第 11 轮深度审计修复(16 项)](#第十一轮第-11-轮深度审计修复)
12. [遗留限制](#遗留限制)
---
@@ -828,7 +829,95 @@
---
## 遗留限制
## 第十一轮:第 11 轮深度审计修复(16 项)
**日期**2026-07-28
**提交**`090938f`
**数量**16 项
**触发**:用户第 6 次要求全面检查
**测试**1007 全部通过 + 前端 build 通过
### P0-1: Keyset 翻页 `title="" or "__NULL__"` 导致下一页空(CRITICAL
- **文件**`search_engine.py:1692-1695`
- **根因**`_cursor_from_item``lit.title or "__NULL__"` — 当 title 为空字符串 `""` 时,Python 的 `or` 短路抛出 `""` 产生 `"__NULL__"` 哨兵值。后续 `_keyset_condition` 生成 `title IS NULL AND id > :uid`,由于 title 有 `NOT NULL` 约束,此条件永远返回零行
- **修复**NOT NULL 列直接使用 `lit.title`(无哨兵);`journal` 列(可为 NULL)保留 `... if ... is not None else "__NULL__"` 而非 `or`
- **同行修复**:相同的 `lit.journal or "__NULL__"` 也修复为 `... if ... is not None else ...`,因为 `""` 是 journal 的合法值,不应被哨兵化
### P1-1: 布尔操作符 `boolean_operator` 受括号内 AND/OR 污染(HIGH
- **文件**`pubmed_query_parser.py:312-322`
- **根因**`boolean_operator` 检测对全 token 流扫描 AND/OR,不区分括号内外。`(A OR B) AND C` 中 OR 在括号内,但代码同时看到 OR 和 AND → 判定为 `"mixed"`(抛出错误)
- **修复**:新增 `depth` 追踪,只在 `depth=0` 时统计 AND/OR
- **验证**`test_complex_nested``test_a4e_double_paren``test_a4e_double_paren_operators``boolean_operator` 预期从 `"mixed"` 修正为 `"and"`
### P1-2: `is_pubmed_syntax()` 误识别英文单词「and/or/not」(HIGH
- **文件**`pubmed_query_parser.py:752`
- **根因**`re.search` 使用 `re.IGNORECASE` 标志。`"diet and exercise in cancer"` 中的 `and` 被识别为 PubMed 布尔符 → 触发 PubMed 路径 → 删除 "and"、"in" 等 stop words → 搜索结果恶化
- **修复**:移除 `re.IGNORECASE`。PubMed 官方仅识别**大写** `AND/OR/NOT` 为布尔符
- **验证**`test_lowercase_boolean_detected` 断言从 `assert is_pubmed_syntax` 改为 `assert not is_pubmed_syntax`
### P1-3: `_relevance_query` 对 MeSH-only 查询为空(HIGH
- **文件**`search_engine.py:605-612`
- **根因**`" ".join(plain_parts).strip() or ""``breast[MAJR]` 这类纯 MeSH 查询的 `plain_parts` 为空,`_relevance_query` 返回 `""``best_match` 路径不会对 tsvector 排序 → 退化到 date sort
- **修复**:改为 `"...".strip() or query`,保留原始查询作为相关性排序回退
- **影响**:修复后 MeSH-only 查询的正确相关性排序工作
### P1-4: `pmid_terms` 缺少 int() 异常处理(HIGH
- **文件**`search_engine.py:1229-1233`
- **根因**`pmid_terms` 处理路径将文本直接 `int(term.text)`,非数字 PMID 格式(如 DOI 格式内容)导致 `ValueError` 崩溃
- **修复**:添加 `try/except ValueError` + DOI ILIKE 兜底,匹配 `_single_term_condition` 已有的模式
- **验证**`10.1000/xyz[PMID]` 这类非数字输入不再崩溃
### P1-5: 部分日期 `YYYY-MM[DP]` 展开为单日而非整月(MEDIUM
- **文件**`pubmed_query_parser.py:408-447`
- **根因**`2024-01[DP]` 被解析器直接当作日期值 `2024-01-01` 处理,范围查询 `2024-01-01:2024-01-01` 只能命中 1 天而非整月
- **修复**:新增 `_PARTIAL_DATE_RE``_expand_partial_date()` 辅助函数,`YYYY-MM` 格式展开为 `YYYY-MM-01:YYYY-MM-31`31 天)
- **影响**:修复 `DP``EDAT``CRDT` 三个字段的部分日期展开
### P1-6: 前端 `#N` 引用重复导致循环引用误判(MEDIUM)
- **文件**`AdvancedPubSearchView.vue:resolveQuery()``useSearchHistory.ts:expandQuery()`
- **根因**:历史引用 `#N` 展开时,若同一个 `N` 在展开列表中多次出现(如同一条 `#1` 在两个位置被引用),`refs` 数组包含重复元素。循环检测逻辑 `refs.includes(ref)` 遇到重复 `#1` 误判为循环
- **修复**:两处都加入 `const uniqueRefs = [...new Set(refs)]` 去重
### P2-1: cache `invalidate_search_cache` 未清理 `filter-options`
- **文件**`cache.py:173`
- **修复**`await self.delete("filter-options")` 加入失效列表
### P2-2: worker 管道异常时缓存未清理
- **文件**`worker.py:28-33,44-50`
- **根因**`daily_ftp_update``daily_citation_update``cache.invalidate_search_cache()` 在正常路径执行,但异常退出时跳过清理
- **修复**`try/finally` 包裹,保证无论成功还是异常都清理搜索缓存
### P2-3: keyset `cursor_val` 空字符串通过 `is None` 检查
- **文件**`search_engine.py:1624`
- **根因**`cursor_val is None or cursor_id is None` — 空字符串 `""` 不满足 `is None`,检查通过,后续 SQL 出错后静默回退到 OFFSET
- **修复**:改为 `not cursor_val or cursor_id is None`
### P2-4: 前端模板条件 `sort === 'date'` 硬编码
- **文件**`SearchView.vue:908`
- **根因**:只有 `date` 排序触发 keyset 条件渲染,实际 `KEYSET_COLUMN_SORTS` 包含 `date/cited/title/journal/first_author` 五种
- **修复**`sort === 'date'``KEYSET_SORTS.has(sort)`
### P2-5: `resetAllFilters` 未重置 `showCustomYear`
- **文件**`SearchView.vue`
- **修复**:重置时补充 `showCustomYear.value = false`
### P2-6: `_field_condition("all")` 中文路径遗漏 author/journal ILIKEDOCS ONLY
- **备注**:第 10 轮修复了中文路径加入 author/journal ILIKE,已在文档中补全。代码已正确
---
截至 2026-07-28,剩余 7 项已知限制:
+2 -1
View File
@@ -41,7 +41,8 @@ export function expandQuery(query: string, entries: HistoryEntry[]): string {
if (current === prev) break
const refs = current.match(/#(\d+)/g)
if (refs) {
for (const ref of refs) {
const uniqueRefs = [...new Set(refs)]
for (const ref of uniqueRefs) {
if (seen.has(ref)) return prev // 循环引用 → 返回上次安全结果
seen.add(ref)
}
+2 -1
View File
@@ -553,6 +553,7 @@ function resetAllFilters() {
sort.value = 'date'
booleanOp.value = 'and'
exactPhrase.value = false
showCustomYear.value = false
goToPage(1)
}
@@ -905,7 +906,7 @@ const specialTags = computed(() => filterOptions.value?.special_tags || {})
/>
<!-- Keyset 分页sort=date不做 COUNT纯翻页 -->
<div v-if="searched && sort === 'date' && results.length > 0" style="display:flex;justify-content:center;align-items:center;gap:12px;padding:20px">
<div v-if="searched && KEYSET_SORTS.has(sort) && results.length > 0" style="display:flex;justify-content:center;align-items:center;gap:12px;padding:20px">
<NButton size="small" :disabled="keysetPage <= 1 || loading" @click="goToPage(keysetPage - 1)"> 上一页</NButton>
<span style="font-size:13px;color:var(--text-muted)"> {{ keysetPage }} </span>
<NButton size="small" :disabled="!keysetHasMore || loading" @click="goToPage(keysetPage + 1)">下一页 </NButton>
@@ -228,7 +228,8 @@ function resolveQuery(q: string): string {
if (current === prev) break
const refs = current.match(/#\d+/g)
if (refs) {
for (const ref of refs) {
const uniqueRefs = [...new Set(refs)]
for (const ref of uniqueRefs) {
if (seen.has(ref)) return prev
seen.add(ref)
}