chore: batch commit remaining changes
CI / backend (push) Canceled after 0s
CI / frontend (push) Canceled after 0s

Includes search engine improvements, Alembic migrations,
new services (pubmed_daily_update, query_expansion),
frontend updates, and documentation sync.
This commit is contained in:
34047007@qq.com
2026-07-27 08:35:12 +08:00
parent 35b0a5c565
commit 62ca8fa6b8
82 changed files with 314248 additions and 1519 deletions
+222 -43
View File
@@ -22,22 +22,58 @@ import re
from dataclasses import dataclass, field
from enum import Enum, auto
# ─── 查询复杂度限制 ───
MAX_TERMS = 50 # 与 API 层 check_query_complexity 的 50 词限制对齐
MAX_PAREN_DEPTH = 10 # 括号嵌套最深层数
# ─── 字段标签映射 ───
_FIELD_TAG_MAP: dict[str, str] = {
"TI": "title",
"AB": "abstract",
"TIAB": "all", # 映射到 field=alltitle+abstract covered
"TIAB": "all",
"AU": "author",
"TA": "journal",
"JT": "journal", # P1-2: Journal Title (同 TA)
"AD": "affiliation",
"CN": "author",
"FAU": "author",
"LAU": "author",
"TW": "all",
"OT": "all",
# 新增标量字段
"LA": "language",
"VI": "volume",
"IP": "issue",
"PG": "pages",
"LID": "lid",
}
# MeSH/PT/DP/PMID/DOI 需要特殊处理不直接映射到 field 参数
_SPECIAL_FIELDS = {"MH", "MAJR", "PT", "DP", "PMID", "DOI"}
# 需要特殊处理的字段(不直接映射到 field 参数
_SPECIAL_FIELDS = {
"MH", "MAJR", "PT", "DP", "PMID", "DOI",
# 日期字段
"EDAT", "CRDT", "MHDA", "LR", "DCOM",
# JSONB 字段
"GR", "SH", "RN", "NM", "SI", "PA",
# 待补充抽取的字段(标记为特殊以便未来实现)
"AUID", "COIS", "ED", "IR", "PS", "PUBN", "TT",
}
# 所有合法字段标签
_ALL_FIELD_TAGS = set(_FIELD_TAG_MAP.keys()) | _SPECIAL_FIELDS
# 支持日期范围语法的字段
_DATE_RANGE_FIELDS = {"DP", "EDAT", "CRDT", "MHDA", "LR", "DCOM"}
# 所有合法字段标签(PubMed 全量字段)
# P1-1: 移除了 BOOK/FILTER/ISBN(未实现,降级为 plain text 不如报错透明)
_ALL_FIELD_TAGS = {
"AB", "AD", "AU", "CN", "FAU", "AUID", "LAU", "COIS",
"DCOM", "CRDT", "EDAT", "MHDA", "LR", "DP", "DOI",
"RN", "ED", "GR", "IR", "IP",
"TA", "JT", "LA", "LID", "MAJR", "SH", "MH", "MH:NOEXP", "OT", "PG",
"PA", "PT", "PMID", "PUBN", "SI", "PS", "NM", "TW",
"TI", "TIAB", "TT", "VI",
}
# ─── Token Types ───
@@ -53,6 +89,8 @@ class TokenType(Enum):
COLON = auto() # :
WORD = auto() # unquoted text
NUMBER = auto() # digits (for PMID, year)
DATE = auto() # YYYY-MM-DD (normalized from YYYY/MM/DD)
UNKNOWN_FIELD = auto() # [...] 但内容不是已知字段标签
EOF = auto() # end of input
@@ -67,12 +105,14 @@ class Token:
_TOKEN_PATTERNS: list[tuple[TokenType, str]] = [
(TokenType.QUOTED, r'"(?:[^"\\]|\\.)*"'),
(TokenType.FIELD, r'\[(?:' + '|'.join(_ALL_FIELD_TAGS) + r')\]'),
(TokenType.UNKNOWN_FIELD, r'\[[^\]]*\]'), # 未识别的字段标签 → 触发降级
(TokenType.AND, r'\bAND\b'),
(TokenType.OR, r'\bOR\b'),
(TokenType.NOT, r'\bNOT\b'),
(TokenType.LPAREN, r'\('),
(TokenType.RPAREN, r'\)'),
(TokenType.COLON, r':'),
(TokenType.DATE, r'\d{4}-\d{2}-\d{2}'),
(TokenType.NUMBER, r'\d+'),
(TokenType.WORD, r'[^\s"\[\]():]+'),
]
@@ -90,7 +130,12 @@ def tokenise(query: str) -> list[Token]:
for name, value in m.groupdict().items():
if value is not None:
ttype = TokenType[name]
if ttype == TokenType.UNKNOWN_FIELD:
fname = value.strip('[]').upper()
raise ParseError(f"不认识的字段标签 [{fname}],降级为简单文本搜索")
tokens.append(Token(ttype, value))
if len(tokens) > MAX_TERMS:
raise ParseError(f"查询词过多(超过 {MAX_TERMS} 个),降级为简单文本搜索")
tokens.append(Token(TokenType.EOF))
return tokens
@@ -104,6 +149,8 @@ class Term:
exact: bool = False # True if quoted phrase
field: str | None = None # None = plain (no field tag); "title"/"abstract"/etc for mapped; "MH"/"PT"/etc for special
is_not: bool = False # True if preceded by NOT
group_id: int = -1 # -1 = top-level, >= 0 = parenthesized group index
_noexp: bool = False # P1-4: [MH:noexp] 抑制树展开
@dataclass
@@ -119,14 +166,54 @@ class ParsedPubmedQuery:
pub_types: list[str] = field(default_factory=list) # [PT]
doi_terms: list[str] = field(default_factory=list) # [DOI]
pmid_terms: list[int] = field(default_factory=list) # [PMID]
affiliation_terms: list[Term] = field(default_factory=list) # [AD]
language_terms: list[Term] = field(default_factory=list) # [LA]
volume_terms: list[Term] = field(default_factory=list) # [VI]
issue_terms: list[Term] = field(default_factory=list) # [IP]
pages_terms: list[Term] = field(default_factory=list) # [PG]
lid_terms: list[Term] = field(default_factory=list) # [LID]
date_from: str | None = None # [DP] lower bound (YYYY-MM-DD)
date_to: str | None = None # [DP] upper bound (YYYY-MM-DD)
year_from: int | None = None # [DP] year lower
year_to: int | None = None # [DP] year upper
# 更多日期字段
edat_from: str | None = None # [EDAT]
edat_to: str | None = None
crdt_from: str | None = None # [CRDT]
crdt_to: str | None = None
mhda_from: str | None = None # [MHDA]
mhda_to: str | None = None
lr_from: str | None = None # [LR]
lr_to: str | None = None
dcom_from: str | None = None # [DCOM]
dcom_to: str | None = None
# 特殊字段(存储 Term 以保留 is_not 标志)
mesh_terms: list[Term] = field(default_factory=list) # [MH] (was list[str])
majr_terms: list[Term] = field(default_factory=list) # [MAJR] (was list[str])
pub_types: list[Term] = field(default_factory=list) # [PT] (was list[str])
doi_terms: list[Term] = field(default_factory=list) # [DOI] (was list[str])
pmid_terms: list[Term] = field(default_factory=list) # [PMID] (was list[int])
# JSONB 字段
grant_terms: list[Term] = field(default_factory=list) # [GR]
subheading_terms: list[Term] = field(default_factory=list) # [SH]
registry_terms: list[Term] = field(default_factory=list) # [RN]
substance_terms: list[Term] = field(default_factory=list) # [NM]
databank_terms: list[Term] = field(default_factory=list) # [SI]
pharmaco_terms: list[Term] = field(default_factory=list) # [PA]
ed_terms: list[Term] = field(default_factory=list) # [ED]
investigator_terms: list[Term] = field(default_factory=list) # [IR]
personal_name_terms: list[Term] = field(default_factory=list) # [PS]
pubnote_terms: list[Term] = field(default_factory=list) # [PUBN]
auid_terms: list[Term] = field(default_factory=list) # [AUID]
cois_terms: list[Term] = field(default_factory=list) # [COIS]
tt_terms: list[Term] = field(default_factory=list) # [TT]
plain_terms: list[Term] = field(default_factory=list) # no field tag
boolean_operator: str = "and" # "and" | "or" | "mixed"
has_not: bool = False # contains NOT
not_terms: list[Term] = field(default_factory=list) # terms under NOT
groups: list[list[Term]] = field(default_factory=list) # parenthesized sub-groups
group_operators: list[str] = field(default_factory=list) # "and"/"or" per group (P2-2)
negated_date_ranges: set[str] = field(default_factory=set) # date fields negated by NOT
# ─── Parser ───
@@ -174,6 +261,7 @@ class PubmedQueryParser:
def parse(self) -> ParsedPubmedQuery:
"""入口:解析完整的查询字符串。"""
result = ParsedPubmedQuery()
self._depth = 0 # 括号嵌套深度计数器
try:
terms = self._parse_or_expr(result)
except ParseError:
@@ -187,12 +275,12 @@ class PubmedQueryParser:
elif has_or and not has_and:
result.boolean_operator = "or"
result.has_not = any(t.is_not for t in terms if not getattr(t, '_is_range_end', False))
result.not_terms = [t for t in terms if t.is_not and not getattr(t, '_is_range_end', False)]
for t in terms:
if getattr(t, '_is_range_end', False):
continue
# 分组词不从 flat lists 走,避免括号内外的词被一起 AND/OR
# 同时 has_not/not_terms 也只考虑非分组词
_ungrouped = [t for t in terms if not getattr(t, '_is_range_end', False) and t.group_id < 0]
result.has_not = any(t.is_not for t in _ungrouped)
result.not_terms = [t for t in _ungrouped if t.is_not]
for t in _ungrouped:
self._dispatch_term(result, t)
return result
@@ -206,26 +294,75 @@ class PubmedQueryParser:
result.tiab_terms.append(term)
elif term.field == "author":
result.author_terms.append(term)
elif term.field == "affiliation":
result.affiliation_terms.append(term)
elif term.field == "journal":
result.journal_terms.append(term)
elif term.field == "language":
result.language_terms.append(term)
elif term.field == "volume":
result.volume_terms.append(term)
elif term.field == "issue":
result.issue_terms.append(term)
elif term.field == "pages":
result.pages_terms.append(term)
elif term.field == "lid":
result.lid_terms.append(term)
elif term.field == "MH":
result.mesh_terms.append(term.text)
result.mesh_terms.append(term)
elif term.field == "MAJR":
result.majr_terms.append(term.text)
result.majr_terms.append(term)
elif term.field == "PT":
result.pub_types.append(term.text)
result.pub_types.append(term)
elif term.field == "PMID":
try:
result.pmid_terms.append(int(term.text))
int(term.text) # validate
except ValueError:
result.plain_terms.append(term)
return
result.pmid_terms.append(term)
elif term.field == "DOI":
result.doi_terms.append(term.text)
result.doi_terms.append(term)
elif term.field == "GR":
result.grant_terms.append(term)
elif term.field == "SH":
result.subheading_terms.append(term)
elif term.field == "RN":
result.registry_terms.append(term)
elif term.field == "NM":
result.substance_terms.append(term)
elif term.field == "SI":
result.databank_terms.append(term)
elif term.field == "PA":
result.pharmaco_terms.append(term)
elif term.field is None:
result.plain_terms.append(term)
elif term.field == "__RANGE_DP__":
# Already handled inline during parse; skip.
pass
elif term.field == "__RANGE_EDAT__":
pass
elif term.field == "__RANGE_CRDT__":
pass
elif term.field == "__RANGE_MHDA__":
pass
elif term.field == "__RANGE_LR__":
pass
elif term.field == "__RANGE_DCOM__":
pass
elif term.field == "ED":
result.ed_terms.append(term)
elif term.field == "IR":
result.investigator_terms.append(term)
elif term.field == "PS":
result.personal_name_terms.append(term)
elif term.field == "PUBN":
result.pubnote_terms.append(term)
elif term.field == "AUID":
result.auid_terms.append(term)
elif term.field == "COIS":
result.cois_terms.append(term)
elif term.field == "TT":
result.tt_terms.append(term)
else:
result.plain_terms.append(term)
@@ -240,7 +377,7 @@ class PubmedQueryParser:
def _is_primary_start(self, token: Token) -> bool:
"""Check if token could start a primary expression."""
return token.type in (TokenType.WORD, TokenType.QUOTED, TokenType.NUMBER,
return token.type in (TokenType.WORD, TokenType.QUOTED, TokenType.NUMBER, TokenType.DATE,
TokenType.LPAREN, TokenType.NOT)
def _parse_and_expr(self, result: ParsedPubmedQuery) -> list[Term]:
@@ -272,9 +409,23 @@ class PubmedQueryParser:
def _parse_primary(self, result: ParsedPubmedQuery, negated: bool = False) -> list[Term]:
"""primary → atom FIELD? | LPAREN query RPAREN"""
if self.peek().type == TokenType.LPAREN:
self._depth += 1
if self._depth > MAX_PAREN_DEPTH:
raise ParseError(f"括号嵌套过深(超过 {MAX_PAREN_DEPTH} 层),降级为简单文本搜索")
self.advance()
start_pos = self.pos # P2-2: 记录组起始标记位置
terms = self._parse_or_expr(result)
end_pos = self.pos # P2-2: 记录组结束标记位置
self.expect(TokenType.RPAREN)
self._depth -= 1
# 标记为子组,不放入 flat lists,保留括号分组结构
group_id = len(result.groups)
for t in terms:
t.group_id = group_id
result.groups.append(terms)
# P2-2: 检测组内是否有显式 OR
_has_or = any(t.type == TokenType.OR for t in self.tokens[start_pos:end_pos])
result.group_operators.append("or" if _has_or else "and")
if negated:
for t in terms:
t.is_not = True
@@ -289,17 +440,19 @@ class PubmedQueryParser:
- "quoted phrase"[FIELD]
- word[FIELD]
- NUMBER:NUMBER[DP] (year/date range)
- DATE:DATE[DP] (full date range like 2024-01-01:2024-12-31)
- NUMBER (bare number)
- word (bare word)
"""
# Look ahead for NUMBER:NUMBER pattern
# Look ahead for range pattern (NUMBER:NUMBER or DATE:DATE)
t0 = self.peek()
t1 = self.peek_n(1)
t2 = self.peek_n(2)
if (t0.type == TokenType.NUMBER
and t1 is not None and t1.type == TokenType.COLON
and t2 is not None and t2.type == TokenType.NUMBER):
if (t1 is not None and t1.type == TokenType.COLON
and t2 is not None
and t0.type in (TokenType.NUMBER, TokenType.DATE, TokenType.WORD)
and t2.type in (TokenType.NUMBER, TokenType.DATE, TokenType.WORD)):
return self._parse_range(result, negated)
# Normal atom
@@ -307,16 +460,23 @@ class PubmedQueryParser:
text = token.value.strip('"') if token.type == TokenType.QUOTED else token.value
is_exact = (token.type == TokenType.QUOTED)
field = None
_noexp = False # P1-4
if self.peek().type == TokenType.FIELD:
ft = self.advance()
field = ft.value[1:-1].upper()
if field in _FIELD_TAG_MAP:
field = _FIELD_TAG_MAP[field]
raw = ft.value[1:-1].upper()
# P1-4: [MH:noexp] → 抑制树展开
if raw == "MH:NOEXP":
field = "MH"
_noexp = True
else:
field = raw
if field in _FIELD_TAG_MAP:
field = _FIELD_TAG_MAP[field]
return [Term(text, exact=is_exact, field=field, is_not=negated)]
return [Term(text, exact=is_exact, field=field, is_not=negated, _noexp=_noexp)]
def _parse_range(self, result: ParsedPubmedQuery, negated: bool = False) -> list[Term]:
"""Parse NUMBER:NUMBER[FIELD] — handles DP (year) ranges specially."""
"""Parse NUMBER:NUMBER[FIELD] — handles date ranges specially."""
start_val = self.advance().value # NUMBER
self.advance() # COLON
end_val = self.advance().value # NUMBER
@@ -326,25 +486,39 @@ class PubmedQueryParser:
ft = self.advance()
field = ft.value[1:-1].upper()
if field == "DP":
try:
sy, ey = int(start_val), int(end_val)
except ValueError:
sy = ey = None
# Year-only range (e.g., 2024:2026[DP])
if field in _DATE_RANGE_FIELDS:
attr_map = {
"DP": ("date_from", "date_to", "year_from", "year_to", "__RANGE_DP__"),
"EDAT": ("edat_from", "edat_to", None, None, "__RANGE_EDAT__"),
"CRDT": ("crdt_from", "crdt_to", None, None, "__RANGE_CRDT__"),
"MHDA": ("mhda_from", "mhda_to", None, None, "__RANGE_MHDA__"),
"LR": ("lr_from", "lr_to", None, None, "__RANGE_LR__"),
"DCOM": ("dcom_from", "dcom_to", None, None, "__RANGE_DCOM__"),
}
date_attr, date_attr_to, yr_from_attr, yr_to_attr, marker_field = attr_map[field]
# 反向范围自动交换(如 2026:2024[DP] → 2024:2026[DP]
if start_val.isdigit() and end_val.isdigit() and int(start_val) > int(end_val):
start_val, end_val = end_val, start_val
# Year-only range (e.g., 2024:2026[EDAT])
if start_val.isdigit() and len(start_val) == 4:
result.year_from = sy
result.year_to = ey
if yr_from_attr:
setattr(result, yr_from_attr, int(start_val))
setattr(result, yr_to_attr, int(end_val))
else:
# For non-DP date fields: convert year to full date for consistency
setattr(result, date_attr, f"{start_val}-01-01")
setattr(result, date_attr_to, f"{end_val}-12-31")
else:
# Full date range (e.g., 2024/01/01:2024/12/31[DP])
result.date_from = start_val
result.date_to = end_val
# Return a marker term so not-terms tracking knows about it, but _dispatch skips.
marker = Term(f"{start_val}:{end_val}", field="__RANGE_DP__", is_not=negated)
# Full date range (e.g., 2024-01-01:2024-12-31[EDAT])
setattr(result, date_attr, start_val)
setattr(result, date_attr_to, end_val)
marker = Term(f"{start_val}:{end_val}", field=marker_field, is_not=negated)
marker._is_range_end = True
if negated:
result.negated_date_ranges.add(field)
return [marker]
# Non-DP range or no field → plain text
# Non-date range or no field → plain text
txt = f"{start_val}:{end_val}"
if field:
txt = f"{txt}[{field}]"
@@ -364,7 +538,7 @@ def is_pubmed_syntax(query: str) -> bool:
return False
if re.search(r'\[(' + '|'.join(_ALL_FIELD_TAGS) + r')\]', query, re.IGNORECASE):
return True
if re.search(r'\b(AND|OR|NOT)\b', query):
if re.search(r'\b(AND|OR|NOT)\b', query, re.IGNORECASE):
return True
return False
@@ -378,6 +552,9 @@ def parse_pubmed_query(query: str) -> ParsedPubmedQuery:
return ParsedPubmedQuery()
try:
# 将 YYYY/MM/DD 格式的日期分隔符统一为 YYYY-MM-DD,使 tokeniser 正确识别为 DATE
import re as _re
query = _re.sub(r'(\d{4})/(\d{2})/(\d{2})', r'\1-\2-\3', query)
tokens = tokenise(query)
parser = PubmedQueryParser(tokens)
return parser.parse()
@@ -392,7 +569,9 @@ def extract_pubmed_query_for_prisma(query: str) -> tuple[str, list[str]]:
(normalized_query, mesh_terms_used)
"""
parsed = parse_pubmed_query(query)
mesh_used = list(set(parsed.mesh_terms + parsed.majr_terms))
mesh_used = list(set(
[t.text for t in parsed.mesh_terms] + [t.text for t in parsed.majr_terms]
))
mesh_used.sort()
# 标准化:统一字段大写