fix: 第11轮深度审计修复 — P0 keyset翻页崩溃 + P1布尔符语义/语法误报等16项修复
P0 (1项): - keyset cursor: lit.title or "__NULL__" 空字符串误判为NULL导致后续翻页全空 P1 (6项): - boolean_operator: 仅扫描depth=0的token,括号内AND/OR不再影响顶层操作符判定 - is_pubmed_syntax: 去除AND/OR/NOT检测的IGNORECASE(PubMed仅识别大写布尔符) - _relevance_query: MeSH-only查询(如breast[MAJR])补充原始回退,避免退化到日期排序 - pmid_terms: 缺少try/except int(),补充ValueError+DOI兜底 - 部分日期展开: 2024-01[DP]等YYYY-MM partial date展开为整月范围(DP/EDAT/CRDT) - 前端#N去重: resolveQuery/expandQuery dedup refs防重复引用误判为循环 P2 (9项): - cache: filter-options加入invalidate_search_cache清理 - worker: daily_ftp_update/daily_citation_update异常时finally保证缓存清理 - cursor_val: 空字符串''通过is None检查,补充not cursor_val - keyset UI: 模板条件sort==='date'改为KEYSET_SORTS.has(sort) - resetAllFilters: 补充showCustomYear=false
This commit is contained in:
@@ -170,6 +170,7 @@ class CacheService:
|
||||
await self.delete("search:year_counts:all")
|
||||
await self.delete("journals:map")
|
||||
await self.delete_pattern("atm:*")
|
||||
await self.delete("filter-options")
|
||||
|
||||
async def invalidate_tenant(self, tenant_id: str):
|
||||
await self.delete(f"tenant:{tenant_id}:plan")
|
||||
|
||||
@@ -23,6 +23,19 @@ import unicodedata
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum, auto
|
||||
|
||||
# P11: partial date YYYY-MM pattern, expanded to full month range
|
||||
_PARTIAL_DATE_RE = re.compile(r'^\d{4}-\d{2}$')
|
||||
_LAST_DAY = {1:31, 2:29, 3:31, 4:30, 5:31, 6:30, 7:31, 8:31, 9:30, 10:31, 11:30, 12:31}
|
||||
|
||||
|
||||
def _expand_partial_date(text: str) -> tuple[str, str]:
|
||||
"""Expand YYYY-MM to full month range (YYYY-MM-01 to YYYY-MM-last_day)."""
|
||||
date_from = f"{text}-01"
|
||||
y, m = text.split("-")
|
||||
last_day = _LAST_DAY.get(int(m), 31)
|
||||
date_to = f"{y}-{m}-{last_day}"
|
||||
return date_from, date_to
|
||||
|
||||
# ─── 查询复杂度限制 ───
|
||||
MAX_TERMS = 100 # P4-1: 放宽到 100 词(原 50 词)
|
||||
MAX_PAREN_DEPTH = 10 # 括号嵌套最深层数
|
||||
@@ -308,9 +321,20 @@ class PubmedQueryParser:
|
||||
terms = self._parse_or_expr(result)
|
||||
# 解析错误由 parse_pubmed_query 统一降级处理
|
||||
|
||||
# Detect boolean operator from token stream
|
||||
has_and = any(t.type == TokenType.AND for t in self.tokens)
|
||||
has_or = any(t.type == TokenType.OR for t in self.tokens)
|
||||
# Detect boolean operator from top-level (depth=0) token stream only
|
||||
# P11: 括号内的 AND/OR 不应影响顶层布尔操作符判定
|
||||
depth = 0
|
||||
has_and = has_or = False
|
||||
for t in self.tokens:
|
||||
if t.type == TokenType.LPAREN:
|
||||
depth += 1
|
||||
elif t.type == TokenType.RPAREN:
|
||||
depth -= 1
|
||||
elif depth == 0:
|
||||
if t.type == TokenType.AND:
|
||||
has_and = True
|
||||
elif t.type == TokenType.OR:
|
||||
has_or = True
|
||||
if has_and and has_or:
|
||||
result.boolean_operator = "mixed"
|
||||
elif has_or and not has_and:
|
||||
@@ -398,6 +422,8 @@ class PubmedQueryParser:
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
elif _PARTIAL_DATE_RE.match(term.text):
|
||||
result.date_from, result.date_to = _expand_partial_date(term.text)
|
||||
else:
|
||||
result.date_from = term.text
|
||||
result.date_to = term.text
|
||||
@@ -407,6 +433,8 @@ class PubmedQueryParser:
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
elif _PARTIAL_DATE_RE.match(term.text):
|
||||
result.edat_from, result.edat_to = _expand_partial_date(term.text)
|
||||
else:
|
||||
result.edat_from = term.text
|
||||
result.edat_to = term.text
|
||||
@@ -416,6 +444,8 @@ class PubmedQueryParser:
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
elif _PARTIAL_DATE_RE.match(term.text):
|
||||
result.crdt_from, result.crdt_to = _expand_partial_date(term.text)
|
||||
else:
|
||||
result.crdt_from = term.text
|
||||
result.crdt_to = term.text
|
||||
@@ -749,7 +779,7 @@ def is_pubmed_syntax(query: str) -> bool:
|
||||
query = unicodedata.normalize('NFKC', query)
|
||||
if re.search(r'\[(' + '|'.join(_ALL_FIELD_TAGS) + r')\]', query, re.IGNORECASE):
|
||||
return True
|
||||
if re.search(r'\b(AND|OR|NOT)\b', query, re.IGNORECASE):
|
||||
if re.search(r'\b(AND|OR|NOT)\b', query):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@@ -609,7 +609,9 @@ class AdvancedSearchEngine:
|
||||
plain_parts += [t.text for t in _pubmed_parsed.title_terms]
|
||||
plain_parts += [t.text for t in _pubmed_parsed.abstract_terms]
|
||||
plain_parts += [t.text for t in _pubmed_parsed.tiab_terms]
|
||||
_relevance_query = " ".join(plain_parts) if plain_parts else ""
|
||||
# P11: MeSH-only 查询(如 breast[MAJR])会产生空 plain_parts,
|
||||
# 退回到原始查询字符串保证相关性排序不退化到日期排序
|
||||
_relevance_query = " ".join(plain_parts).strip() or query
|
||||
# ── 通用 keyset 分页(所有列式排序模式统一,代替 OFFSET) ──
|
||||
_keyset_cond = AdvancedSearchEngine._keyset_condition(sort, cursor_val, cursor_id)
|
||||
q = select(GlobalLiterature)
|
||||
@@ -1227,7 +1229,10 @@ class AdvancedSearchEngine:
|
||||
|
||||
# 7. [PMID] → 精确匹配,支持 is_not
|
||||
for term in pp.pmid_terms:
|
||||
cond = GlobalLiterature.pmid == int(term.text)
|
||||
try:
|
||||
cond = GlobalLiterature.pmid == int(term.text)
|
||||
except ValueError:
|
||||
cond = GlobalLiterature.doi.ilike(f"%{_escape_ilike(term.text)}%")
|
||||
if term.is_not:
|
||||
cond = not_(cond)
|
||||
conditions.append(cond)
|
||||
@@ -1621,7 +1626,7 @@ class AdvancedSearchEngine:
|
||||
"""
|
||||
if sort not in AdvancedSearchEngine.KEYSET_COLUMN_SORTS:
|
||||
return None
|
||||
if cursor_val is None or cursor_id is None:
|
||||
if not cursor_val or cursor_id is None:
|
||||
return None
|
||||
import uuid as _uuid
|
||||
try:
|
||||
@@ -1690,9 +1695,9 @@ class AdvancedSearchEngine:
|
||||
return str(lit.cited_by_count)
|
||||
return "__NULL__"
|
||||
elif sort == "title":
|
||||
return lit.title or "__NULL__"
|
||||
return lit.title # title is NOT NULL per model
|
||||
elif sort == "journal":
|
||||
return lit.journal or "__NULL__"
|
||||
return lit.journal if lit.journal is not None else "__NULL__"
|
||||
elif sort == "first_author":
|
||||
authors = lit.authors or []
|
||||
return authors[0].get("family") if authors else "__NULL__"
|
||||
|
||||
@@ -24,11 +24,13 @@ async def shutdown(ctx):
|
||||
|
||||
async def daily_ftp_update(ctx):
|
||||
"""每日 FTP 增量更新(取代旧的精搜+宽搜+retagger)"""
|
||||
stats = await run_daily_ftp_update(ctx)
|
||||
# 搜索缓存失效:管道新增/修改文献后,缓存中的搜索结果立即过时
|
||||
from app.core.cache import cache
|
||||
await cache.invalidate_search_cache()
|
||||
return stats if isinstance(stats, dict) else {"status": "ok"}
|
||||
try:
|
||||
stats = await run_daily_ftp_update(ctx)
|
||||
return stats if isinstance(stats, dict) else {"status": "ok"}
|
||||
finally:
|
||||
# 搜索缓存失效:无论管道成功还是异常,都保证清理
|
||||
await cache.invalidate_search_cache()
|
||||
|
||||
|
||||
async def daily_digest_task(ctx):
|
||||
@@ -40,7 +42,12 @@ async def daily_digest_task(ctx):
|
||||
async def daily_citation_update(ctx):
|
||||
"""每日引用次数更新任务"""
|
||||
from app.services.citation_updater import run_citation_update
|
||||
return await run_citation_update(cached_hours=24)
|
||||
from app.core.cache import cache
|
||||
try:
|
||||
stats = await run_citation_update(cached_hours=24)
|
||||
return stats if isinstance(stats, dict) else {"status": "ok"}
|
||||
finally:
|
||||
await cache.invalidate_search_cache()
|
||||
|
||||
|
||||
async def refresh_hot_articles_cache(ctx):
|
||||
|
||||
@@ -192,13 +192,13 @@ class TestParserGaps:
|
||||
"""((lung OR breast) AND therapy) — nested or flat groups"""
|
||||
r = parse_pubmed_query("((lung OR breast) AND therapy)")
|
||||
assert len(r.groups) >= 1
|
||||
# outer paren: ((a OR b) AND therapy) — should have at least one group
|
||||
assert r.boolean_operator == "mixed"
|
||||
# P11: 括号内的 OR 不影响顶层操作符,顶层全部在括号内 → 无顶层 AND/OR → "and"
|
||||
assert r.boolean_operator == "and"
|
||||
|
||||
def test_a4e_double_paren_operators(self):
|
||||
"""((lung OR breast) AND therapy[TI]) — mixed ops"""
|
||||
r = parse_pubmed_query("((lung OR breast) AND therapy[TI])")
|
||||
assert r.boolean_operator == "mixed"
|
||||
assert r.boolean_operator == "and" # P11: 括号内的 OR 不影响顶层
|
||||
assert len(r.groups) >= 1
|
||||
assert r.has_not is False
|
||||
|
||||
|
||||
@@ -32,8 +32,8 @@ class TestIsPubmedSyntax:
|
||||
assert not is_pubmed_syntax(None)
|
||||
|
||||
def test_lowercase_boolean_detected(self):
|
||||
"""小写的 and/or/not 也识别为 PubMed 语法(P0-1 修复)"""
|
||||
assert is_pubmed_syntax("cancer and therapy")
|
||||
"""小写的 and/or/not 不识别为 PubMed 语法(P11: 仅大写 AND/OR/NOT 才是 PubMed 布尔符)"""
|
||||
assert not is_pubmed_syntax("cancer and therapy")
|
||||
|
||||
|
||||
class TestTokenise:
|
||||
@@ -152,7 +152,8 @@ class TestParsePubmedQuery:
|
||||
# 括号分组后,term 进入 groups 而非直接列在 title_terms
|
||||
assert len(result.groups) > 0
|
||||
assert len(result.tiab_terms) == 1
|
||||
assert result.boolean_operator == "mixed"
|
||||
assert result.boolean_operator == "and"
|
||||
# P11: 括号内的 OR 不影响顶层布尔操作符判定,顶层只有 AND
|
||||
# 确保 groups 中包含括号内的两个 title 词
|
||||
group_texts = [t.text for g in result.groups for t in g]
|
||||
assert "lung cancer" in group_texts
|
||||
|
||||
@@ -172,7 +172,8 @@ class TestAllFrontendQueryFormats:
|
||||
assert r.mesh_terms[0].text == "mouse"
|
||||
assert r.mesh_terms[0].is_not is True
|
||||
assert r.has_not is True
|
||||
assert r.boolean_operator == "mixed"
|
||||
assert r.boolean_operator == "and"
|
||||
# P11: 括号内的 OR 不影响顶层,顶层只有 AND + NOT → boolean_operator="and",NOT 通过 has_not 处理
|
||||
assert len(r.groups) == 1
|
||||
group_texts = [t.text for t in r.groups[0]]
|
||||
assert "lung cancer" in group_texts
|
||||
|
||||
@@ -41,7 +41,8 @@ export function expandQuery(query: string, entries: HistoryEntry[]): string {
|
||||
if (current === prev) break
|
||||
const refs = current.match(/#(\d+)/g)
|
||||
if (refs) {
|
||||
for (const ref of refs) {
|
||||
const uniqueRefs = [...new Set(refs)]
|
||||
for (const ref of uniqueRefs) {
|
||||
if (seen.has(ref)) return prev // 循环引用 → 返回上次安全结果
|
||||
seen.add(ref)
|
||||
}
|
||||
|
||||
@@ -553,6 +553,7 @@ function resetAllFilters() {
|
||||
sort.value = 'date'
|
||||
booleanOp.value = 'and'
|
||||
exactPhrase.value = false
|
||||
showCustomYear.value = false
|
||||
goToPage(1)
|
||||
}
|
||||
|
||||
@@ -905,7 +906,7 @@ const specialTags = computed(() => filterOptions.value?.special_tags || {})
|
||||
/>
|
||||
|
||||
<!-- Keyset 分页(sort=date,不做 COUNT,纯翻页) -->
|
||||
<div v-if="searched && sort === 'date' && results.length > 0" style="display:flex;justify-content:center;align-items:center;gap:12px;padding:20px">
|
||||
<div v-if="searched && KEYSET_SORTS.has(sort) && results.length > 0" style="display:flex;justify-content:center;align-items:center;gap:12px;padding:20px">
|
||||
<NButton size="small" :disabled="keysetPage <= 1 || loading" @click="goToPage(keysetPage - 1)">← 上一页</NButton>
|
||||
<span style="font-size:13px;color:var(--text-muted)">第 {{ keysetPage }} 页</span>
|
||||
<NButton size="small" :disabled="!keysetHasMore || loading" @click="goToPage(keysetPage + 1)">下一页 →</NButton>
|
||||
|
||||
@@ -228,7 +228,8 @@ function resolveQuery(q: string): string {
|
||||
if (current === prev) break
|
||||
const refs = current.match(/#\d+/g)
|
||||
if (refs) {
|
||||
for (const ref of refs) {
|
||||
const uniqueRefs = [...new Set(refs)]
|
||||
for (const ref of uniqueRefs) {
|
||||
if (seen.has(ref)) return prev
|
||||
seen.add(ref)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user