chore: batch commit remaining changes
Includes search engine improvements, Alembic migrations, new services (pubmed_daily_update, query_expansion), frontend updates, and documentation sync.
This commit is contained in:
@@ -0,0 +1,661 @@
|
||||
"""高级搜索服务:布尔运算 + 字段限定 + PubMed 查询语法"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import String, and_, case, cast, func, literal_column, not_, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.literature import GlobalJournal, GlobalLiterature, GlobalLiteratureTag, GlobalTag, GlobalTagTreeNumber
|
||||
from app.schemas.literature import cap_pub_date
|
||||
|
||||
|
||||
class AdvancedSearchEngine:
|
||||
"""PG 高级搜索(ES 就绪后切换 search_service.py)"""
|
||||
|
||||
SEARCH_CACHE_TTL = 300 # 秒(夜间流水线更新,5分钟缓存很安全)
|
||||
|
||||
@staticmethod
|
||||
def _search_cache_key(
|
||||
query: str, field: str, boolean: str, exact_phrase: bool,
|
||||
year_from: int | None, year_to: int | None,
|
||||
date_from: str | None, date_to: str | None,
|
||||
journal_tiers: list[str] | None, pub_types: list[str] | None,
|
||||
study_design: str | None, tag_ids: list[str] | None,
|
||||
retracted: str, negative_result: str,
|
||||
is_oa: bool | None, language: str | None, nlm_subsets: list[str] | None,
|
||||
page: int, page_size: int, sort: str,
|
||||
) -> str:
|
||||
"""归一化查询参数 → 确定性缓存 key(所有 list 排序后参与哈希)"""
|
||||
import hashlib, json
|
||||
norm = {
|
||||
"q": query.strip().lower(),
|
||||
"f": field, "b": boolean, "ep": exact_phrase,
|
||||
"yf": year_from, "yt": year_to,
|
||||
"df": date_from, "dt": date_to,
|
||||
"jt": sorted(journal_tiers) if journal_tiers else [],
|
||||
"pt": sorted(pub_types) if pub_types else [],
|
||||
"sd": study_design,
|
||||
"tid": sorted(tag_ids) if tag_ids else [],
|
||||
"r": retracted, "nr": negative_result,
|
||||
"oa": is_oa, "lang": language,
|
||||
"ns": sorted(nlm_subsets) if nlm_subsets else [],
|
||||
"p": page, "ps": page_size, "s": sort,
|
||||
}
|
||||
raw = json.dumps(norm, sort_keys=True, ensure_ascii=False, default=str)
|
||||
return f"search:advanced:{hashlib.md5(raw.encode()).hexdigest()}"
|
||||
|
||||
@staticmethod
|
||||
async def search(
|
||||
db: AsyncSession,
|
||||
query: str = "",
|
||||
field: str = "all",
|
||||
boolean: str = "and",
|
||||
exact_phrase: bool = False,
|
||||
year_from: int | None = None,
|
||||
year_to: int | None = None,
|
||||
date_from: str | None = None, # YYYY-MM-DD
|
||||
date_to: str | None = None, # YYYY-MM-DD
|
||||
journal_tiers: list[str] | None = None,
|
||||
pub_types: list[str] | None = None,
|
||||
study_design: str | None = None, # primary category: interventional / observational / synthesis / etc
|
||||
tag_ids: list[str] | None = None,
|
||||
retracted: str = "", # "yes", "no", "only"
|
||||
negative_result: str = "", # "yes", "no", "only"
|
||||
is_oa: bool | None = None, # True = 仅开放获取
|
||||
language: str | None = None, # 语言代码(en/zh/fr 等)
|
||||
nlm_subsets: list[str] | None = None, # NLM 期刊子集(AIM/M/S 等)
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
sort: str = "date",
|
||||
cursor_date: str | None = None, # keyset 游标:上一页最后一条的 pub_date
|
||||
cursor_id: str | None = None, # keyset 游标:上一页最后一条的 id(UUID)
|
||||
) -> dict:
|
||||
"""执行高级搜索"""
|
||||
conditions = []
|
||||
|
||||
# 60s 缓存(仅 page 模式,cursor 游标不重复)
|
||||
use_cursor = (cursor_date is not None and cursor_id is not None and sort == "date")
|
||||
_cache = None
|
||||
_search_cache_key = None
|
||||
if not use_cursor:
|
||||
from app.core.cache import cache as _cache
|
||||
_search_cache_key = AdvancedSearchEngine._search_cache_key(
|
||||
query, field, boolean, exact_phrase,
|
||||
year_from, year_to, date_from, date_to,
|
||||
journal_tiers, pub_types, study_design, tag_ids,
|
||||
retracted, negative_result,
|
||||
is_oa, language, nlm_subsets,
|
||||
page, page_size, sort,
|
||||
)
|
||||
cached = await _cache.get(_search_cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
# ─── PubMed 语法检测与解析 ───
|
||||
_pubmed_parsed = None
|
||||
_is_flat_text = True # 是否走传统 tsvector + ILIKE 路径
|
||||
if query.strip():
|
||||
from app.services.pubmed_query_parser import is_pubmed_syntax, parse_pubmed_query
|
||||
|
||||
if is_pubmed_syntax(query):
|
||||
pp = parse_pubmed_query(query)
|
||||
_pubmed_parsed = pp
|
||||
|
||||
# 只要解析器产出了有效结构,就走 PubMed 路径
|
||||
has_pubmed_terms = bool(
|
||||
pp.title_terms or pp.abstract_terms or pp.tiab_terms
|
||||
or pp.author_terms or pp.journal_terms
|
||||
or pp.mesh_terms or pp.majr_terms
|
||||
or pp.pub_types or pp.doi_terms or pp.pmid_terms
|
||||
or pp.plain_terms or pp.has_not
|
||||
or pp.year_from or pp.year_to
|
||||
)
|
||||
if has_pubmed_terms:
|
||||
_is_flat_text = False
|
||||
conditions = await AdvancedSearchEngine._pubmed_conditions(
|
||||
db, pp, conditions,
|
||||
)
|
||||
|
||||
# ─── 传统搜索路径(纯文本 / PubMed 退化) ───
|
||||
if _is_flat_text and query.strip():
|
||||
# 中文搜索:自动匹配 GlobalTag.name_zh → 注入 tag_ids,跳过 ILIKE
|
||||
import re as _re
|
||||
_CHINESE_RE = _re.compile(r'[一-鿿]')
|
||||
if _CHINESE_RE.search(query):
|
||||
tag_matches = (await db.execute(
|
||||
select(GlobalTag.id).where(GlobalTag.name_zh.ilike(f'%{query.strip()}%'))
|
||||
)).scalars().all()
|
||||
if tag_matches:
|
||||
existing = set(tag_ids or [])
|
||||
tag_ids = list(existing | {str(t.id) for t in tag_matches})
|
||||
query = "" # ILIKE 对中文标题/摘要无效,跳过
|
||||
if query.strip():
|
||||
terms = [t.strip() for t in query.split() if t.strip()]
|
||||
# 单数字词:优先 PMID 精确匹配(unique index 5ms 返回)
|
||||
# 不是 PMID 时才回退到 ILIKE 兜底(DOI 片段等),不做 tsquery 避免 seq scan
|
||||
numeric_terms = [t for t in terms if t.isdigit() and len(t) <= 15]
|
||||
text_terms = [t for t in terms if not (t.isdigit() and len(t) <= 15)]
|
||||
for t in numeric_terms:
|
||||
p = int(t)
|
||||
exists = (await db.execute(
|
||||
select(GlobalLiterature.id).where(GlobalLiterature.pmid == p).limit(1)
|
||||
)).scalar_one_or_none()
|
||||
if exists:
|
||||
conditions.append(GlobalLiterature.pmid == p)
|
||||
else:
|
||||
# 不是 PMID → ILIKE 搜索(DOI 等),不做 tsquery 避免 seq scan
|
||||
p_like = t if exact_phrase else f"%{t}%"
|
||||
conditions.append(or_(
|
||||
GlobalLiterature.title.ilike(p_like),
|
||||
GlobalLiterature.doi.ilike(p_like),
|
||||
))
|
||||
if text_terms:
|
||||
# ATM 展开(仅非中文查询)
|
||||
_atm_cond = None
|
||||
if not _CHINESE_RE.search(query):
|
||||
from app.services.query_expansion import expand_atm as _expand_atm
|
||||
_atm_cond = await _expand_atm(db, query.strip())
|
||||
|
||||
_cond_before = len(conditions)
|
||||
if boolean == "and":
|
||||
for term in text_terms:
|
||||
conditions.append(AdvancedSearchEngine._field_condition(field, term, exact_phrase))
|
||||
else:
|
||||
or_conds = [AdvancedSearchEngine._field_condition(field, t, exact_phrase) for t in text_terms]
|
||||
conditions.append(or_(*or_conds))
|
||||
|
||||
# 将文本条件与 ATM 条件 OR 组合
|
||||
if _atm_cond is not None:
|
||||
_text_conds = conditions[_cond_before:]
|
||||
del conditions[_cond_before:]
|
||||
if _text_conds:
|
||||
if len(_text_conds) == 1:
|
||||
conditions.append(or_(_atm_cond, _text_conds[0]))
|
||||
else:
|
||||
conditions.append(or_(_atm_cond, and_(*_text_conds)))
|
||||
else:
|
||||
conditions.append(_atm_cond)
|
||||
|
||||
# 年份范围
|
||||
if year_from:
|
||||
conditions.append(GlobalLiterature.pub_year >= year_from)
|
||||
if year_to:
|
||||
conditions.append(GlobalLiterature.pub_year <= year_to)
|
||||
|
||||
# 具体日期范围(按天搜索)
|
||||
from datetime import date as dt_date
|
||||
if date_from:
|
||||
try:
|
||||
df = dt_date.fromisoformat(date_from)
|
||||
conditions.append(GlobalLiterature.pub_date >= df)
|
||||
except ValueError:
|
||||
raise ValueError(f"Invalid date_from format: {date_from}")
|
||||
if date_to:
|
||||
try:
|
||||
dt = dt_date.fromisoformat(date_to)
|
||||
conditions.append(GlobalLiterature.pub_date <= dt)
|
||||
except ValueError:
|
||||
raise ValueError(f"Invalid date_to format: {date_to}")
|
||||
|
||||
# 期刊等级
|
||||
if journal_tiers:
|
||||
subq = select(GlobalJournal.issn).where(GlobalJournal.tier.in_(journal_tiers))
|
||||
result = await db.execute(subq)
|
||||
issns = [r for (r,) in result.all()]
|
||||
if issns:
|
||||
conditions.append(GlobalLiterature.journal_issn.in_(issns))
|
||||
|
||||
# 标签筛选(含子标签递归)
|
||||
if tag_ids:
|
||||
import uuid as _uuid
|
||||
tag_uuids = [_uuid.UUID(tid) if isinstance(tid, str) else tid for tid in tag_ids]
|
||||
all_tags = (await db.execute(select(GlobalTag).where(GlobalTag.id.in_(tag_uuids)))).scalars().all()
|
||||
all_tag_ids = set(t.id for t in all_tags if t)
|
||||
# Batch child tag lookup (1 query instead of N)
|
||||
paths = [t.path + "::" for t in all_tags if t]
|
||||
if paths:
|
||||
child_conds = [GlobalTag.path.like(f"{p}%") for p in paths]
|
||||
children = (await db.execute(select(GlobalTag.id).where(or_(*child_conds)))).scalars().all()
|
||||
all_tag_ids.update(children)
|
||||
uids = list(all_tag_ids)
|
||||
subq = select(func.distinct(GlobalLiteratureTag.literature_id)).where(GlobalLiteratureTag.tag_id.in_(uids))
|
||||
conditions.append(GlobalLiterature.id.in_(subq))
|
||||
|
||||
# 发表类型(PG JSONB contains)
|
||||
if pub_types:
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
type_conds = [GlobalLiterature.pub_types.cast(JSONB).contains([pt]) for pt in pub_types]
|
||||
conditions.append(or_(*type_conds))
|
||||
|
||||
# 研究设计分类过滤(PG JSON path access)
|
||||
if study_design:
|
||||
conditions.append(
|
||||
GlobalLiterature.study_design['primary'].astext == study_design
|
||||
)
|
||||
|
||||
# 撤稿过滤
|
||||
if retracted == "yes":
|
||||
conditions.append(GlobalLiterature.retracted == True)
|
||||
elif retracted == "no":
|
||||
conditions.append(GlobalLiterature.retracted == False)
|
||||
elif retracted == "only":
|
||||
conditions.append(GlobalLiterature.retracted == True)
|
||||
|
||||
# 阴性结果过滤
|
||||
if negative_result == "yes":
|
||||
conditions.append(GlobalLiterature.is_negative_result == True)
|
||||
elif negative_result == "no":
|
||||
conditions.append(GlobalLiterature.is_negative_result == False)
|
||||
elif negative_result == "only":
|
||||
conditions.append(GlobalLiterature.is_negative_result == True)
|
||||
|
||||
# 开放获取
|
||||
if is_oa is not None:
|
||||
conditions.append(GlobalLiterature.is_oa == is_oa)
|
||||
|
||||
# 语言
|
||||
if language:
|
||||
conditions.append(GlobalLiterature.language == language)
|
||||
|
||||
# NLM 期刊子集(如 AIM / Core Clinical Journals)
|
||||
if nlm_subsets:
|
||||
subq = select(GlobalJournal.issn).where(GlobalJournal.nlm_subsets.overlap(nlm_subsets))
|
||||
result = await db.execute(subq)
|
||||
issns = [r for (r,) in result.all()]
|
||||
if issns:
|
||||
conditions.append(GlobalLiterature.journal_issn.in_(issns))
|
||||
|
||||
# ── 按年份统计(Results by year,与筛选条件一致) ──
|
||||
year_counts = []
|
||||
if conditions and (query.strip() or year_from or year_to or date_from or date_to
|
||||
or journal_tiers or pub_types or study_design or tag_ids
|
||||
or retracted or negative_result or is_oa is not None
|
||||
or language or nlm_subsets):
|
||||
try:
|
||||
yr_subq = select(GlobalLiterature.pub_year).where(
|
||||
and_(*conditions)
|
||||
).subquery()
|
||||
year_count_q = select(
|
||||
yr_subq.c.pub_year, func.count().label("cnt")
|
||||
).group_by(yr_subq.c.pub_year).order_by(yr_subq.c.pub_year.desc())
|
||||
year_rows = await db.execute(year_count_q)
|
||||
year_counts = [
|
||||
{"year": y, "count": c} for y, c in year_rows if y is not None
|
||||
]
|
||||
except Exception:
|
||||
year_counts = [] # 构建查询
|
||||
q = select(GlobalLiterature)
|
||||
if conditions:
|
||||
q = q.where(and_(*conditions))
|
||||
|
||||
# 排序(PubMed 查询时跳过 ts_rank,避免语法标签噪音)
|
||||
_relevance_query = query
|
||||
if _pubmed_parsed and sort == "relevance":
|
||||
# 用纯文本词做相关性排序,去掉 [field] 标签
|
||||
plain_parts = [t.text for t in _pubmed_parsed.plain_terms]
|
||||
plain_parts += [t.text for t in _pubmed_parsed.title_terms]
|
||||
plain_parts += [t.text for t in _pubmed_parsed.tiab_terms]
|
||||
_relevance_query = " ".join(plain_parts) if plain_parts else ""
|
||||
if sort == "date":
|
||||
q = q.order_by(GlobalLiterature.pub_date.desc().nullslast())
|
||||
elif sort == "cited":
|
||||
q = q.order_by(GlobalLiterature.cited_by_count.desc().nullslast())
|
||||
elif sort == "best_match" and _relevance_query.strip():
|
||||
tsq = func.plainto_tsquery("english", _relevance_query)
|
||||
q = q.order_by(AdvancedSearchEngine._best_match_order(tsq))
|
||||
elif sort == "relevance" and _relevance_query.strip():
|
||||
tsq = func.plainto_tsquery("english", _relevance_query)
|
||||
rank = func.ts_rank(GlobalLiterature.search_tsv, tsq)
|
||||
q = q.order_by(rank.desc())
|
||||
else:
|
||||
q = q.order_by(GlobalLiterature.pub_date.desc().nullslast())
|
||||
|
||||
# keyset 游标分页(仅 date 排序支持,page 参数被忽略,不做 COUNT)
|
||||
use_keyset = (cursor_date is not None and cursor_id is not None and sort == "date")
|
||||
if use_keyset:
|
||||
from datetime import date as dt_date
|
||||
try:
|
||||
cursor_dt = dt_date.fromisoformat(cursor_date)
|
||||
except (ValueError, TypeError):
|
||||
use_keyset = False
|
||||
if use_keyset:
|
||||
import uuid as _cuuid
|
||||
try:
|
||||
cursor_uuid = _cuuid.UUID(cursor_id)
|
||||
conditions.append(
|
||||
or_(
|
||||
GlobalLiterature.pub_date < cursor_dt,
|
||||
and_(GlobalLiterature.pub_date == cursor_dt, GlobalLiterature.id < cursor_uuid),
|
||||
)
|
||||
)
|
||||
except (ValueError, AttributeError):
|
||||
use_keyset = False
|
||||
|
||||
# 重新构建查询(keyset 条件可能已追加)
|
||||
q = select(GlobalLiterature)
|
||||
if conditions:
|
||||
q = q.where(and_(*conditions))
|
||||
if sort == "date":
|
||||
q = q.order_by(GlobalLiterature.pub_date.desc().nullslast())
|
||||
elif sort == "cited":
|
||||
q = q.order_by(GlobalLiterature.cited_by_count.desc().nullslast())
|
||||
elif sort == "best_match" and _relevance_query.strip():
|
||||
tsq = func.plainto_tsquery("english", _relevance_query)
|
||||
q = q.order_by(AdvancedSearchEngine._best_match_order(tsq))
|
||||
elif sort == "relevance" and _relevance_query.strip():
|
||||
tsq = func.plainto_tsquery("english", _relevance_query)
|
||||
rank = func.ts_rank(GlobalLiterature.search_tsv, tsq)
|
||||
q = q.order_by(rank.desc())
|
||||
else:
|
||||
q = q.order_by(GlobalLiterature.pub_date.desc().nullslast())
|
||||
|
||||
# 分页
|
||||
has_more = False
|
||||
if use_keyset:
|
||||
# cursor 模式:取 page_size+1 条判断是否有下一页,不做 COUNT
|
||||
result = await db.execute(q.limit(page_size + 1))
|
||||
items = result.scalars().all()
|
||||
has_more = len(items) > page_size
|
||||
items = items[:page_size]
|
||||
total = 0
|
||||
else:
|
||||
# 总数
|
||||
count_q = select(func.count()).select_from(
|
||||
select(literal_column("1"))
|
||||
.select_from(GlobalLiterature)
|
||||
.where(and_(*conditions) if conditions else True)
|
||||
.subquery()
|
||||
)
|
||||
total = (await db.execute(count_q)).scalar() or 0
|
||||
offset = (page - 1) * page_size
|
||||
|
||||
# ── date 排序优化 ──
|
||||
# PG 对常见词(如 "cancer" >80% 匹配率)会选 Parallel Seq Scan(~22s)。
|
||||
# 嵌入子查询先取最近 N 条 ID(走 ix_gl_pub_date 索引),迫使 PG
|
||||
# 使用索引而非全表扫描。
|
||||
# 条件:仅当无可搜索查询且非历史日期筛选时启用,避免丢失早期文献
|
||||
_has_meaningful_query = bool(query.strip())
|
||||
_has_historical_filter = year_from is not None and year_from < datetime.now().year - 5
|
||||
if sort == "date" and conditions and not _has_meaningful_query and not _has_historical_filter:
|
||||
recent_n = max(offset + page_size * 5, 2000)
|
||||
recent_subq = select(GlobalLiterature.id).where(
|
||||
GlobalLiterature.pub_date.isnot(None)
|
||||
).order_by(GlobalLiterature.pub_date.desc()).limit(recent_n)
|
||||
data_q = select(GlobalLiterature).where(
|
||||
GlobalLiterature.id.in_(recent_subq),
|
||||
and_(*conditions)
|
||||
).order_by(GlobalLiterature.pub_date.desc().nullslast())
|
||||
else:
|
||||
data_q = q
|
||||
|
||||
result = await db.execute(data_q.offset(offset).limit(page_size))
|
||||
items = result.scalars().all()
|
||||
has_more = (offset + page_size) < total and len(items) == page_size
|
||||
|
||||
from app.services.tag_loader import load_tags_for_literature
|
||||
tm = await load_tags_for_literature(db, [str(lit.id) for lit in items])
|
||||
|
||||
# Batch load journal tiers + canonical names
|
||||
tier_map = {}
|
||||
name_map = {}
|
||||
if items:
|
||||
issns = list(set(lit.journal_issn for lit in items if lit.journal_issn))
|
||||
if issns:
|
||||
jr = await db.execute(
|
||||
select(GlobalJournal.issn, GlobalJournal.tier, GlobalJournal.name).where(GlobalJournal.issn.in_(issns))
|
||||
)
|
||||
for issn_, tier_, name_ in jr.all():
|
||||
tier_map[issn_] = tier_
|
||||
name_map[issn_] = name_
|
||||
|
||||
results = []
|
||||
for lit in items:
|
||||
authors = lit.authors or []
|
||||
results.append({
|
||||
"id": str(lit.id), "pmid": lit.pmid, "title": lit.title,
|
||||
"first_author": authors[0].get("family", "") if authors else "",
|
||||
"journal": name_map.get(lit.journal_issn) or lit.journal, "pub_date": cap_pub_date(lit.pub_date),
|
||||
"article_date": lit.article_date.isoformat() if lit.article_date else None,
|
||||
"pub_year": lit.pub_year, "tags": tm.get(str(lit.id), []),
|
||||
"abstract": lit.abstract[:300] if lit.abstract else None,
|
||||
"doi": lit.doi,
|
||||
"pmc_id": lit.pmc_id,
|
||||
"is_oa": lit.is_oa,
|
||||
"cited_by_count": lit.cited_by_count,
|
||||
"created_at": lit.created_at.isoformat() if lit.created_at else None,
|
||||
"updated_at": lit.updated_at.isoformat() if lit.updated_at else None,
|
||||
"journal_issn": lit.journal_issn,
|
||||
"journal_tier": tier_map.get(lit.journal_issn),
|
||||
"pub_types": lit.pub_types,
|
||||
"affiliation": authors[0].get("affiliation", "") if authors else "",
|
||||
})
|
||||
|
||||
result = {"items": results, "total": total, "page": page, "page_size": page_size, "has_more": has_more, "year_counts": year_counts}
|
||||
|
||||
if _search_cache_key is not None:
|
||||
await _cache.set(_search_cache_key, result, ttl=AdvancedSearchEngine.SEARCH_CACHE_TTL)
|
||||
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
async def _pubmed_conditions(
|
||||
db: AsyncSession,
|
||||
pp,
|
||||
existing_conditions: list,
|
||||
) -> list:
|
||||
"""将解析后的 PubMed 查询转换为 SQLAlchemy 条件列表。"""
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
conditions = list(existing_conditions)
|
||||
|
||||
# 1. 字段级搜索 [TI] [AB] [TIAB] [AU] [TA]
|
||||
field_map = {
|
||||
"title": pp.title_terms,
|
||||
"abstract": pp.abstract_terms,
|
||||
"all": pp.tiab_terms,
|
||||
"author": pp.author_terms,
|
||||
"journal": pp.journal_terms,
|
||||
}
|
||||
for fld, terms in field_map.items():
|
||||
if not terms:
|
||||
continue
|
||||
field_conds = []
|
||||
for term in terms:
|
||||
cond = AdvancedSearchEngine._field_condition(fld, term.text, term.exact)
|
||||
if term.is_not:
|
||||
cond = not_(cond)
|
||||
field_conds.append(cond)
|
||||
# 同 field 内多个 term 按 boolean_operator 组合
|
||||
if pp.boolean_operator == "or":
|
||||
conditions.append(or_(*field_conds))
|
||||
else:
|
||||
conditions.extend(field_conds) # and_() 由外部统一组合
|
||||
|
||||
# 2. 纯文本词(无字段标签)
|
||||
if pp.plain_terms:
|
||||
if pp.boolean_operator == "or":
|
||||
plain_conds = []
|
||||
for term in pp.plain_terms:
|
||||
cond = AdvancedSearchEngine._field_condition("all", term.text, term.exact)
|
||||
if term.is_not:
|
||||
cond = not_(cond)
|
||||
plain_conds.append(cond)
|
||||
if plain_conds:
|
||||
conditions.append(or_(*plain_conds))
|
||||
else:
|
||||
for term in pp.plain_terms:
|
||||
cond = AdvancedSearchEngine._field_condition("all", term.text, term.exact)
|
||||
if term.is_not:
|
||||
cond = not_(cond)
|
||||
conditions.append(cond)
|
||||
|
||||
# 3. [MH] → tree_number 展开(逐 term 独立 subq,AND 组合)
|
||||
if pp.mesh_terms:
|
||||
if pp.boolean_operator == "or":
|
||||
conditions = await AdvancedSearchEngine._expand_mesh_tag_ids(
|
||||
db, pp.mesh_terms, conditions)
|
||||
else:
|
||||
for m in pp.mesh_terms:
|
||||
conditions = await AdvancedSearchEngine._expand_mesh_tag_ids(
|
||||
db, [m], conditions)
|
||||
|
||||
# 4. [MAJR] → tree_number 展开 + is_major=True(逐 term 独立 subq)
|
||||
if pp.majr_terms:
|
||||
if pp.boolean_operator == "or":
|
||||
conditions = await AdvancedSearchEngine._expand_mesh_tag_ids(
|
||||
db, pp.majr_terms, conditions, major_only=True)
|
||||
else:
|
||||
for m in pp.majr_terms:
|
||||
conditions = await AdvancedSearchEngine._expand_mesh_tag_ids(
|
||||
db, [m], conditions, major_only=True)
|
||||
|
||||
# 5. [PT] → pub_types JSONB contains
|
||||
if pp.pub_types:
|
||||
type_conds = [
|
||||
GlobalLiterature.pub_types.cast(JSONB).contains([pt])
|
||||
for pt in pp.pub_types
|
||||
]
|
||||
conditions.append(or_(*type_conds))
|
||||
|
||||
# 5. [DP] → 年份/日期范围
|
||||
if pp.year_from:
|
||||
conditions.append(GlobalLiterature.pub_year >= pp.year_from)
|
||||
if pp.year_to:
|
||||
conditions.append(GlobalLiterature.pub_year <= pp.year_to)
|
||||
if pp.date_from:
|
||||
from datetime import date as _dt_date
|
||||
try:
|
||||
conditions.append(GlobalLiterature.pub_date >= _dt_date.fromisoformat(pp.date_from))
|
||||
except ValueError:
|
||||
pass
|
||||
if pp.date_to:
|
||||
from datetime import date as _dt_date
|
||||
try:
|
||||
conditions.append(GlobalLiterature.pub_date <= _dt_date.fromisoformat(pp.date_to))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# 6. [PMID] → 精确匹配
|
||||
for pmid_val in pp.pmid_terms:
|
||||
conditions.append(GlobalLiterature.pmid == pmid_val)
|
||||
|
||||
# 7. [DOI] → ILIKE
|
||||
for doi_term in pp.doi_terms:
|
||||
conditions.append(GlobalLiterature.doi.ilike(f"%{doi_term}%"))
|
||||
|
||||
return conditions
|
||||
|
||||
@staticmethod
|
||||
def _field_condition(field: str, term: str, exact: bool) -> callable:
|
||||
if field == "title":
|
||||
if exact:
|
||||
return GlobalLiterature.title.ilike(term if exact else f"%{term}%")
|
||||
# 只用 tsquery(GIN 索引),不掺 ILIKE——OR 会让 PG 放弃 GIN 索引走 seq scan
|
||||
return GlobalLiterature.search_tsv.op("@@")(func.plainto_tsquery("english", term))
|
||||
elif field == "abstract":
|
||||
if exact:
|
||||
return GlobalLiterature.abstract.ilike(term if exact else f"%{term}%")
|
||||
return GlobalLiterature.search_tsv.op("@@")(func.plainto_tsquery("english", term))
|
||||
elif field == "author":
|
||||
return GlobalLiterature.search_tsv.op("@@")(func.plainto_tsquery("english", term))
|
||||
elif field == "journal":
|
||||
return or_(
|
||||
GlobalLiterature.journal.ilike(f"%{term}%"),
|
||||
GlobalLiterature.journal_iso.ilike(f"%{term}%"),
|
||||
)
|
||||
elif field == "affiliation":
|
||||
return GlobalLiterature.search_tsv.op("@@")(func.plainto_tsquery("english", term))
|
||||
else: # "all" default
|
||||
like_val = term if exact else f"%{term}%"
|
||||
if exact:
|
||||
return or_(
|
||||
GlobalLiterature.title.ilike(like_val),
|
||||
cast(GlobalLiterature.pmid, String).ilike(f"%{term}%"),
|
||||
GlobalLiterature.doi.ilike(like_val),
|
||||
)
|
||||
# 含 "/" → 跳过 tsquery
|
||||
if "/" in term:
|
||||
if term.startswith("10."):
|
||||
return or_(
|
||||
GlobalLiterature.doi == term,
|
||||
GlobalLiterature.doi.ilike(like_val),
|
||||
)
|
||||
return or_(
|
||||
GlobalLiterature.title.ilike(like_val),
|
||||
cast(GlobalLiterature.pmid, String).ilike(f"%{term}%"),
|
||||
GlobalLiterature.doi.ilike(like_val),
|
||||
)
|
||||
# 只用 tsquery(GIN 索引),不掺 ILIKE——OR 会让 PG 放弃 GIN 索引走 seq scan
|
||||
return GlobalLiterature.search_tsv.op("@@")(func.plainto_tsquery("english", term))
|
||||
|
||||
@staticmethod
|
||||
async def _expand_mesh_tag_ids(
|
||||
db: AsyncSession,
|
||||
mesh_names: list[str],
|
||||
conditions: list,
|
||||
major_only: bool = False,
|
||||
) -> list:
|
||||
"""用 GlobalTagTreeNumber 展开 [MH]/[MAJR]
|
||||
|
||||
1. ILIKE 匹配 mesh_names → tag_ids
|
||||
2. 用 tree_number 前缀展开子节点(C04.588 → 所有子树号)
|
||||
3. [MAJR] 额外 AND is_major=True
|
||||
"""
|
||||
import uuid as _uuid
|
||||
mesh_conds = [GlobalTag.name_en.ilike(f"%{m}%") for m in mesh_names]
|
||||
tag_rows = (await db.execute(
|
||||
select(GlobalTag.id).where(or_(*mesh_conds))
|
||||
)).all()
|
||||
if not tag_rows:
|
||||
return conditions
|
||||
|
||||
mesh_tag_ids: set[_uuid.UUID] = set(tid for (tid,) in tag_rows)
|
||||
|
||||
# tree_number 前缀展开:取匹配 tag 的所有 tree_number,查子节点
|
||||
tns = (await db.execute(
|
||||
select(GlobalTagTreeNumber.tree_number).where(
|
||||
GlobalTagTreeNumber.tag_id.in_(list(mesh_tag_ids))
|
||||
).distinct()
|
||||
)).scalars().all()
|
||||
|
||||
if tns:
|
||||
child_conds = [GlobalTagTreeNumber.tree_number.like(f"{tn}%") for tn in tns]
|
||||
children = (await db.execute(
|
||||
select(GlobalTagTreeNumber.tag_id).where(or_(*child_conds))
|
||||
)).scalars().all()
|
||||
mesh_tag_ids.update(children)
|
||||
|
||||
uids = list(mesh_tag_ids)
|
||||
if major_only:
|
||||
subq = select(func.distinct(GlobalLiteratureTag.literature_id)).where(
|
||||
GlobalLiteratureTag.tag_id.in_(uids),
|
||||
GlobalLiteratureTag.is_major == True,
|
||||
)
|
||||
else:
|
||||
subq = select(func.distinct(GlobalLiteratureTag.literature_id)).where(
|
||||
GlobalLiteratureTag.tag_id.in_(uids),
|
||||
)
|
||||
conditions.append(GlobalLiterature.id.in_(subq))
|
||||
return conditions
|
||||
|
||||
@staticmethod
|
||||
def _best_match_order(tsq):
|
||||
"""构建 best_match 排序表达式:ts_rank + 引用数对数 + 近期度梯度加分
|
||||
|
||||
近期度加分通过 EXTRACT(YEAR FROM NOW()) 动态计算,分三档:
|
||||
- 当年: +10
|
||||
- 去年: +7
|
||||
- 前年: +3
|
||||
"""
|
||||
_cy = func.extract("year", func.now())
|
||||
score = (
|
||||
func.ts_rank(GlobalLiterature.search_tsv, tsq) * 0.3
|
||||
+ func.ln(func.coalesce(GlobalLiterature.cited_by_count, 0) + 1) * 2
|
||||
+ case(
|
||||
(GlobalLiterature.pub_year >= _cy, 10),
|
||||
(GlobalLiterature.pub_year >= _cy - 1, 7),
|
||||
(GlobalLiterature.pub_year >= _cy - 2, 3),
|
||||
else_=0,
|
||||
)
|
||||
)
|
||||
return score.desc()
|
||||
Reference in New Issue
Block a user