fix: 第8轮搜索深度审计修复 — 缓存失效、Redis重试、中文标签、翻页稳定性等12项
CRITICAL:
- invalidate_search_cache 清理 atm:* 缓存(MeSH ATM扩展不再用过期结果)
- Pro 方案 api_quota_per_day 1000→10000(修复低于 Free 的数据错误)
- CacheService/RateLimitMiddleware Redis 连接失败60秒自动重试(原永久降级)
- 普通搜索中文输入自动匹配 GlobalTag.name_zh(如"肺癌"通过MeSH标签关联文献)
- AdvancedPubSearchView resolveQuery 添加 seen Set 检测交替 #N 循环引用
HIGH:
- 限速器 _burst_windows 每500请求清理过期条目(防止内存泄漏)
- cron daily_ftp_update 末尾调用 invalidate_search_cache()(自动管道不再用过期缓存)
MEDIUM:
- _apply_order_by ASC 排序加 id tiebreaker(title/journal/first_author翻页跳行/重复)
- _keyset_condition 所有 is_(None) 加 id tiebreaker + __NULL__ 哨兵值
- _field_condition("all") 默认tsvector路径加 journal/journal_iso ILIKE 兜底
- SearchView restoreFromQuery date_preset/year_from/year_to 优先顺序修复
docs: 更新 12/13 搜索文档,移除 CLAUDE.md 陈旧 SQLite 提及
This commit is contained in:
@@ -167,7 +167,6 @@ curl -X POST "localhost:8000/api/v1/admin/pipeline/refresh-citations?limit=500"
|
||||
|
||||
- **No multi-specialty in one deployment.** One codebase, one YAML config, one Docker stack per specialty. Oncology is the first.
|
||||
- **Personal → Team upgrade is zero-data-migration.** Personal user's `tenant_id` stays the same; only `is_personal` flips and `plan_type` changes.
|
||||
- **SQLite in dev, PostgreSQL in prod.** SQLAlchemy generic types enable this.
|
||||
- **Dev mode password reset** returns the reset link directly in API response (no SMTP needed).
|
||||
- **`user["sub"]` is a string.** Always convert to `uuid.UUID()` before passing to SQLAlchemy queries.
|
||||
- **搜索功能必须与 PubMed 完全一致。** 这是硬性要求,不是"未来优化"。所有 PubMed 字段标签必须全量支持,已存储数据的立即接通搜索路径,缺失数据的补充 XML 抽取和存储。不允许任何字段退化到纯文本搜索。
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""backfill tag_ids from global_literature_tag
|
||||
|
||||
Revision ID: b01b8f27c596
|
||||
Revises: e0764f6d7c21
|
||||
Create Date: 2026-07-27 22:51:25.288622
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = 'b01b8f27c596'
|
||||
down_revision: Union[str, None] = 'e0764f6d7c21'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# 回填 tag_ids:从 global_literature_tags 聚合存量数据
|
||||
# 注:author_names_text 的回填已在 f1a2b3c4d5e6 中完成,无需重复
|
||||
#
|
||||
# 用 array_agg + JOIN 代替关联子查询,避免每行触发子查询。
|
||||
# 对百万/千万级数据线性扩展(hash 聚合 + hash join),预期 2-5 分钟。
|
||||
op.execute("""
|
||||
UPDATE global_literature gl
|
||||
SET tag_ids = agg.tag_ids
|
||||
FROM (
|
||||
SELECT literature_id, array_agg(tag_id ORDER BY tag_id) AS tag_ids
|
||||
FROM global_literature_tags
|
||||
GROUP BY literature_id
|
||||
) agg
|
||||
WHERE gl.id = agg.literature_id
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# 回滚:清空 tag_ids(列为 nullable=True,回填是幂等的)
|
||||
op.execute("""
|
||||
UPDATE global_literature SET tag_ids = NULL
|
||||
WHERE tag_ids IS NOT NULL
|
||||
""")
|
||||
@@ -67,11 +67,11 @@ def upgrade() -> None:
|
||||
op.add_column('global_literature', sa.Column('entrez_date', sa.DateTime(timezone=True), nullable=True))
|
||||
|
||||
# ─── 2. 19 列 JSON → JSONB(单笔 ALTER TABLE,一次全表重写)───
|
||||
for col in _JSON_TO_JSONB_COLS:
|
||||
op.alter_column('global_literature', col,
|
||||
existing_type=postgresql.JSON(astext_type=sa.Text()),
|
||||
type_=postgresql.JSONB(astext_type=sa.Text()),
|
||||
existing_nullable=True)
|
||||
_cols = ", ".join(
|
||||
f"ALTER COLUMN \"{c}\" TYPE jsonb USING \"{c}\"::jsonb"
|
||||
for c in _JSON_TO_JSONB_COLS
|
||||
)
|
||||
op.execute(f"ALTER TABLE global_literature {_cols}")
|
||||
|
||||
# ─── 3. entry_terms 到 global_tags ───
|
||||
op.add_column('global_tags', sa.Column('entry_terms', postgresql.JSONB(astext_type=sa.Text()), nullable=True))
|
||||
|
||||
@@ -53,33 +53,6 @@ _TSVEC_OLD = """setweight(to_tsvector('english', COALESCE(NEW.title, '')), 'A')
|
||||
'')
|
||||
), 'A')"""
|
||||
|
||||
# 回填用 UPDATE 表达式(引用列名而非 NEW)
|
||||
_UPDATE_SQL = """UPDATE global_literature
|
||||
SET search_tsv = setweight(to_tsvector('english', COALESCE(title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', COALESCE(abstract, '')), 'B') ||
|
||||
setweight(to_tsvector('simple',
|
||||
COALESCE(
|
||||
(SELECT string_agg(
|
||||
value->>'family' || ' ' || COALESCE(value->>'affiliation', ''),
|
||||
' ')
|
||||
FROM jsonb_array_elements(authors)),
|
||||
'')
|
||||
), 'A') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value->>'name', ' ')
|
||||
FROM jsonb_array_elements(chemical_list)),
|
||||
'')
|
||||
), 'C') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(value #>> '{}', ' ')
|
||||
FROM jsonb_array_elements(gene_symbols)),
|
||||
'')
|
||||
), 'C')
|
||||
WHERE chemical_list IS NOT NULL AND chemical_list != '[]'::jsonb
|
||||
OR gene_symbols IS NOT NULL AND gene_symbols != '[]'::jsonb"""
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# 1. 先删除触发器(否则函数不能替换)
|
||||
@@ -105,9 +78,8 @@ def upgrade() -> None:
|
||||
EXECUTE FUNCTION update_literature_search_tsv()
|
||||
""")
|
||||
|
||||
# 4. 回填已有数据的 search_tsv(仅更新有 chemical/gene 的行,减少 I/O)
|
||||
op.execute(_UPDATE_SQL)
|
||||
|
||||
# 注意:此处省略 search_tsv 回填。下一个 migration(g0h1i2j3k4l5)会做全量回填,
|
||||
# 包含 chemical/gene 公式,此处的部分回填是冗余的。
|
||||
|
||||
def downgrade() -> None:
|
||||
# 1. 删除触发器
|
||||
|
||||
@@ -1565,6 +1565,12 @@ async def _run_ftp_pipeline(db: AsyncSession) -> dict:
|
||||
await refresh_stats_cache()
|
||||
except Exception:
|
||||
pass
|
||||
# 管道运行后失效搜索缓存(新文献立即可见)
|
||||
from app.core.cache import cache
|
||||
try:
|
||||
await cache.invalidate_search_cache()
|
||||
except Exception:
|
||||
logger.exception("搜索缓存失效失败")
|
||||
return stats
|
||||
|
||||
|
||||
@@ -1601,6 +1607,11 @@ async def _run_eutils_pipeline(db: AsyncSession, mode: str, max_per_query: int,
|
||||
await refresh_stats_cache()
|
||||
except Exception:
|
||||
pass
|
||||
from app.core.cache import cache
|
||||
try:
|
||||
await cache.invalidate_search_cache()
|
||||
except Exception:
|
||||
logger.exception("搜索缓存失效失败")
|
||||
return stats
|
||||
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from app.models.user import User, UserSavedFilter
|
||||
from app.services.daily_digest import send_daily_digest_to_user
|
||||
from app.services.rule_engine import PRESET_RULES, RuleEngine
|
||||
from app.services.search_engine import AdvancedSearchEngine
|
||||
from app.core.constants import AGE_GROUPS, AGE_GROUP_UI_MAP
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -138,27 +139,6 @@ SPECIAL_MESH_UIS = {
|
||||
],
|
||||
}
|
||||
|
||||
# AGE 筛选层级定义:(key, label, mesh_uis, indent)
|
||||
AGE_GROUPS = [
|
||||
("child_0_18", "Child: birth-18 years", ["D007231", "D007223", "D002675", "D002648", "D000293"], 0),
|
||||
("newborn", "Newborn: birth-1 month", ["D007231"], 1),
|
||||
("infant_0_23", "Infant: birth-23 months", ["D007231", "D007223"], 1),
|
||||
("infant_1_23", "Infant: 1-23 months", ["D007223"], 1),
|
||||
("preschool", "Preschool Child: 2-5 years", ["D002675"], 1),
|
||||
("child_6_12", "Child: 6-12 years", ["D002648"], 1),
|
||||
("adolescent", "Adolescent: 13-18 years", ["D000293"], 1),
|
||||
("adult_19_plus", "Adult: 19+ years", ["D000328", "D008875", "D000368", "D000369"], 0),
|
||||
("young_adult", "Young Adult: 19-24 years", ["D055815"], 1),
|
||||
("adult_19_44", "Adult: 19-44 years", ["D000328"], 1),
|
||||
("middle_aged_plus","Middle Aged + Aged: 45+ years", ["D008875", "D000368", "D000369"], 1),
|
||||
("middle_aged", "Middle Aged: 45-64 years", ["D008875"], 1),
|
||||
("aged", "Aged: 65+ years", ["D000368", "D000369"], 1),
|
||||
("aged_80_plus", "80 and over: 80+ years", ["D000369"], 1),
|
||||
]
|
||||
|
||||
# 预构建 key → UIs 查找表(搜索时用)
|
||||
AGE_GROUP_UI_MAP: dict[str, list[str]] = {k: u for k, _, u, _ in AGE_GROUPS}
|
||||
|
||||
NLM_SUBSET_LABELS = {
|
||||
"AIM": "Core clinical journals",
|
||||
"M": "MEDLINE",
|
||||
@@ -274,7 +254,7 @@ async def advanced_search(req: AdvancedSearchRequest, db: AsyncSession = Depends
|
||||
return await AdvancedSearchEngine.search(db, **req.model_dump())
|
||||
except ValueError as ve:
|
||||
logger.warning("搜索参数错误: %s", ve)
|
||||
raise HTTPException(status_code=400, detail="搜索参数错误,请检查输入")
|
||||
raise HTTPException(status_code=400, detail=str(ve))
|
||||
except Exception as e:
|
||||
logger.exception("搜索服务内部错误")
|
||||
raise HTTPException(status_code=500, detail="搜索服务内部错误") from e
|
||||
|
||||
@@ -259,7 +259,18 @@ async def search_literature(
|
||||
return {"items": [], "total": 0}
|
||||
if len(q.split()) > 100:
|
||||
return {"items": [], "total": 0, "error": "查询词过多(最多 100 个词),请简化搜索条件"}
|
||||
# PubMed 语法检测与降级:普通搜索不支持字段标签和布尔符
|
||||
from app.services.pubmed_query_parser import is_pubmed_syntax
|
||||
if is_pubmed_syntax(q):
|
||||
import re as _pm_re
|
||||
q = _pm_re.sub(r'\[[\w/: -]+\]', '', q)
|
||||
q = _pm_re.sub(r'\b(AND|OR|NOT)\b', '', q)
|
||||
q = q.replace('"', '').replace('(', '').replace(')', '')
|
||||
q = ' '.join(q.split())
|
||||
if not q.strip():
|
||||
return {"items": [], "total": 0}
|
||||
offset = (page - 1) * page_size
|
||||
await db.execute(text("SET LOCAL statement_timeout = '30s'"))
|
||||
like = f"%{_escape_ilike(q)}%"
|
||||
# tsvector 主搜索 + ILIKE 兜底
|
||||
search_cond = or_(
|
||||
@@ -267,6 +278,21 @@ async def search_literature(
|
||||
GlobalLiterature.title.ilike(like),
|
||||
GlobalLiterature.abstract.ilike(like),
|
||||
)
|
||||
# 中文搜索:自动匹配 GlobalTag.name_zh → 注入标签条件
|
||||
import re as _cn_re
|
||||
_CHINESE_RE = _cn_re.compile(r'[一-鿿㐀-䶿豈-]')
|
||||
if _CHINESE_RE.search(q):
|
||||
_tag_matches = (await db.execute(
|
||||
select(GlobalTag.id).where(
|
||||
GlobalTag.source.in_(["mesh", "manual"]),
|
||||
GlobalTag.name_zh.ilike(like),
|
||||
).limit(100)
|
||||
)).scalars().all()
|
||||
if _tag_matches:
|
||||
_tag_lit_subq = select(GlobalLiteratureTag.literature_id).where(
|
||||
GlobalLiteratureTag.tag_id.in_([str(t) for t in _tag_matches])
|
||||
)
|
||||
search_cond = or_(search_cond, GlobalLiterature.id.in_(_tag_lit_subq))
|
||||
count_q = select(func.count(GlobalLiterature.id)).where(search_cond)
|
||||
total = (await db.execute(count_q)).scalar() or 0
|
||||
tsq = func.plainto_tsquery("english", q)
|
||||
|
||||
@@ -18,9 +18,12 @@ class CacheService:
|
||||
self._redis = None
|
||||
|
||||
async def _get_redis(self):
|
||||
"""尝试获取 Redis 连接,失败即降级(只尝试一次)"""
|
||||
"""尝试获取 Redis 连接,失败即降级(每 60 秒重试一次)"""
|
||||
if self._redis_failed:
|
||||
return None
|
||||
import time as _time
|
||||
if getattr(self, '_redis_retry_at', 0) > _time.monotonic():
|
||||
return None
|
||||
self._redis_failed = False
|
||||
if self._redis is not None:
|
||||
return self._redis
|
||||
try:
|
||||
@@ -33,7 +36,9 @@ class CacheService:
|
||||
except Exception:
|
||||
self._redis_failed = True
|
||||
self._redis = None
|
||||
logger.info("Redis 不可用,使用内存缓存")
|
||||
import time as _time
|
||||
self._redis_retry_at = _time.monotonic() + 60
|
||||
logger.info("Redis 不可用,使用内存缓存(60 秒后重试)")
|
||||
return None
|
||||
|
||||
async def get(self, key: str) -> dict | None:
|
||||
@@ -158,9 +163,41 @@ class CacheService:
|
||||
async def invalidate_user(self, user_id: str):
|
||||
await self.delete(f"user:{user_id}:profile")
|
||||
|
||||
async def invalidate_search_cache(self):
|
||||
"""管道运行后失效所有搜索相关缓存。"""
|
||||
await self.delete_pattern("search:advanced:*")
|
||||
await self.delete_pattern("search:year_counts:*")
|
||||
await self.delete("search:year_counts:all")
|
||||
await self.delete("journals:map")
|
||||
await self.delete_pattern("atm:*")
|
||||
|
||||
async def invalidate_tenant(self, tenant_id: str):
|
||||
await self.delete(f"tenant:{tenant_id}:plan")
|
||||
await self.delete(f"tenant:{tenant_id}:settings")
|
||||
|
||||
async def delete_pattern(self, pattern: str):
|
||||
"""Delete all keys matching a glob pattern.
|
||||
|
||||
Redis 模式用 SCAN 0 MATCH pattern 迭代删除。
|
||||
内存模式用 OrderedDict key 前缀匹配删除。
|
||||
"""
|
||||
r = await self._get_redis()
|
||||
if r:
|
||||
try:
|
||||
cursor = 0
|
||||
while True:
|
||||
cursor, keys = await r.scan(cursor, match=pattern, count=100)
|
||||
if keys:
|
||||
await r.delete(*keys)
|
||||
if cursor == 0:
|
||||
break
|
||||
except Exception:
|
||||
logger.exception("Redis SCAN/DEL failed for pattern: %s", pattern)
|
||||
else:
|
||||
import fnmatch
|
||||
to_delete = [k for k in self._store if fnmatch.fnmatch(k, pattern)]
|
||||
for k in to_delete:
|
||||
self._store.pop(k, None)
|
||||
|
||||
|
||||
cache = CacheService()
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Shared constants used across the application."""
|
||||
|
||||
# ─── AGE 筛选层级定义 ───
|
||||
|
||||
AGE_GROUPS = [
|
||||
("child_0_18", "Child: birth-18 years", ["D007231", "D007223", "D002675", "D002648", "D000293"], 0),
|
||||
("newborn", "Newborn: birth-1 month", ["D007231"], 1),
|
||||
("infant_0_23", "Infant: birth-23 months", ["D007231", "D007223"], 1),
|
||||
("infant_1_23", "Infant: 1-23 months", ["D007223"], 1),
|
||||
("preschool", "Preschool Child: 2-5 years", ["D002675"], 1),
|
||||
("child_6_12", "Child: 6-12 years", ["D002648"], 1),
|
||||
("adolescent", "Adolescent: 13-18 years", ["D000293"], 1),
|
||||
("adult_19_plus", "Adult: 19+ years", ["D000328", "D008875", "D000368", "D000369"], 0),
|
||||
("young_adult", "Young Adult: 19-24 years", ["D055815"], 1),
|
||||
("adult_19_44", "Adult: 19-44 years", ["D000328"], 1),
|
||||
("middle_aged_plus","Middle Aged + Aged: 45+ years", ["D008875", "D000368", "D000369"], 1),
|
||||
("middle_aged", "Middle Aged: 45-64 years", ["D008875"], 1),
|
||||
("aged", "Aged: 65+ years", ["D000368", "D000369"], 1),
|
||||
("aged_80_plus", "80 and over: 80+ years", ["D000369"], 1),
|
||||
]
|
||||
|
||||
# 预构建 key → UIs 查找表(搜索时用)
|
||||
AGE_GROUP_UI_MAP: dict[str, list[str]] = {k: u for k, _, u, _ in AGE_GROUPS}
|
||||
@@ -34,7 +34,7 @@ PLANS = {
|
||||
"max_literature_items": 10_000,
|
||||
"max_saved_items": 10_000,
|
||||
"max_review_inclusions": 500,
|
||||
"api_quota_per_day": 1_000,
|
||||
"api_quota_per_day": 10_000,
|
||||
"teams": False,
|
||||
"approval_workflows": False,
|
||||
"sso": False,
|
||||
|
||||
@@ -16,6 +16,8 @@ from app.core.tenant_context import tenant_ctx
|
||||
BURST_MAX_PER_SECOND = 30
|
||||
# 滑动窗口时长(秒)
|
||||
BURST_WINDOW = 1
|
||||
# 每 N 次请求清理一次过期突发窗口
|
||||
BURST_CLEANUP_INTERVAL = 500
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -33,10 +35,15 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
|
||||
self._lock = asyncio.Lock()
|
||||
# 突发保护:{tenant_id: [(timestamp, ...)]}
|
||||
self._burst_windows: dict[str, list[float]] = defaultdict(list)
|
||||
self._burst_cleanup_counter = 0
|
||||
|
||||
async def _get_redis(self):
|
||||
if self._redis is not None:
|
||||
return self._redis
|
||||
if self._redis is False:
|
||||
import time as _rt
|
||||
if getattr(self, '_redis_retry_at', 0) > _rt.monotonic():
|
||||
return None
|
||||
try:
|
||||
from redis.asyncio import Redis
|
||||
r = Redis.from_url(settings.REDIS_URL, decode_responses=True, socket_connect_timeout=1)
|
||||
@@ -44,6 +51,8 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
|
||||
self._redis = r
|
||||
except Exception:
|
||||
self._redis = False
|
||||
import time as _rt
|
||||
self._redis_retry_at = _rt.monotonic() + 60
|
||||
return self._redis if self._redis else None
|
||||
|
||||
async def _get_quota(self, tenant_id: str) -> int:
|
||||
@@ -84,10 +93,30 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
|
||||
window.append(now)
|
||||
return True
|
||||
|
||||
def _cleanup_stale_burst_windows(self):
|
||||
"""清理过期突发窗口条目,防止 _burst_windows 无限增长。"""
|
||||
now = time.time()
|
||||
cutoff = now - BURST_WINDOW
|
||||
stale_keys = []
|
||||
for tid, window in list(self._burst_windows.items()):
|
||||
# 移除过期时间戳
|
||||
while window and window[0] < cutoff:
|
||||
window.pop(0)
|
||||
if not window:
|
||||
stale_keys.append(tid)
|
||||
for k in stale_keys:
|
||||
del self._burst_windows[k]
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
if not request.url.path.startswith("/api/v1/"):
|
||||
return await call_next(request)
|
||||
|
||||
# 定期清理过期突发窗口条目,防止内存泄漏
|
||||
self._burst_cleanup_counter += 1
|
||||
if self._burst_cleanup_counter >= BURST_CLEANUP_INTERVAL:
|
||||
self._burst_cleanup_counter = 0
|
||||
self._cleanup_stale_burst_windows()
|
||||
|
||||
tid = tenant_ctx.get()
|
||||
if not tid:
|
||||
auth_header = request.headers.get("Authorization", "")
|
||||
|
||||
@@ -221,7 +221,7 @@ async def get_funnel(db: AsyncSession, tenant_id: uuid.UUID | None = None, days:
|
||||
|
||||
|
||||
async def get_retention(db: AsyncSession, tenant_id: uuid.UUID | None = None, weeks: int = 12):
|
||||
"""周级留存 cohort 分析(兼容 SQLite + PostgreSQL)"""
|
||||
"""周级留存 cohort 分析(PostgreSQL)"""
|
||||
from collections import defaultdict
|
||||
|
||||
since = datetime.now(UTC) - timedelta(weeks=weeks * 2)
|
||||
|
||||
@@ -22,6 +22,13 @@ from pathlib import Path
|
||||
# C04 树前缀(Neoplasms 及其子类)
|
||||
C04_TREE_PREFIXES = ("C04", "C04.")
|
||||
|
||||
# 交叉引入树号(非 C04 但高度肿瘤相关的 MeSH)
|
||||
CROSS_INCLUDE_PREFIXES = (
|
||||
"E02.319", # Antineoplastic agents — 抗癌药
|
||||
"E02.815", # Radiotherapy — 放疗
|
||||
"D27.505.954.248", # Antineoplastic agents (D27 subclass) — 抗肿瘤药
|
||||
)
|
||||
|
||||
# 标题/摘要关键词(全部生效,修复了之前 keywords[:3] 的 bug)
|
||||
ONCOLOGY_KEYWORDS = [
|
||||
"cancer", "carcinoma", "tumor", "tumour", "neoplasm", "malignancy",
|
||||
@@ -71,12 +78,20 @@ class MeshFilter:
|
||||
return self._tree_map.get(descriptor_name.strip().lower(), [])
|
||||
|
||||
def is_oncology_by_mesh(self, mesh_headings: list[dict]) -> bool:
|
||||
"""按 MeSH 判断是否肿瘤相关(基于 C04 树号)"""
|
||||
"""按 MeSH 判断是否肿瘤相关(C04 树号 + 交叉引入树号)
|
||||
|
||||
交叉引入树号覆盖:
|
||||
E02.319 — Antineoplastic agents(抗癌药)
|
||||
E02.815 — Radiotherapy(放疗)
|
||||
D27.505.954.248 — Antineoplastic agents(D27 子类,抗肿瘤药)
|
||||
"""
|
||||
for heading in mesh_headings:
|
||||
name = heading.get("descriptor", "") or heading.get("name", "")
|
||||
trees = self.get_tree_numbers(name)
|
||||
if any(t.startswith(C04_TREE_PREFIXES) for t in trees):
|
||||
return True
|
||||
if any(t.startswith(CROSS_INCLUDE_PREFIXES) for t in trees):
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -242,9 +242,11 @@ def _parse_europe_pmc_article(art: dict) -> dict | None:
|
||||
"descriptor": desc, "ui": ui, "major": major, "qualifiers": qualifiers,
|
||||
})
|
||||
|
||||
# Keywords
|
||||
keywords = [kw for kw in
|
||||
(art.get("keywordList", {}).get("keyword") or []) if kw]
|
||||
# Keywords(Europe PMC 可能返回单字符串而非数组)
|
||||
_kw_val = art.get("keywordList", {}).get("keyword")
|
||||
if isinstance(_kw_val, str):
|
||||
_kw_val = [_kw_val]
|
||||
keywords = [kw for kw in (_kw_val or []) if kw]
|
||||
|
||||
# Publication types
|
||||
pub_types = []
|
||||
|
||||
@@ -392,40 +392,69 @@ class PubmedQueryParser:
|
||||
elif term.field is None:
|
||||
result.plain_terms.append(term)
|
||||
# ── 独立日期字段(非范围语法):"2024-01-01"[DP] → from=to=该日期 ──
|
||||
# 纯 4 位年份 "2024"[DP] 用 year_from/year_to,避免 fromisoformat 问题
|
||||
# is_not 时加入 negated_date_ranges,引擎据此 NOT 条件
|
||||
elif term.field == "DP":
|
||||
result.date_from = term.text
|
||||
result.date_to = term.text
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
else:
|
||||
result.date_from = term.text
|
||||
result.date_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("DP")
|
||||
elif term.field == "EDAT":
|
||||
result.edat_from = term.text
|
||||
result.edat_to = term.text
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
else:
|
||||
result.edat_from = term.text
|
||||
result.edat_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("EDAT")
|
||||
elif term.field == "CRDT":
|
||||
result.crdt_from = term.text
|
||||
result.crdt_to = term.text
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
else:
|
||||
result.crdt_from = term.text
|
||||
result.crdt_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("CRDT")
|
||||
elif term.field == "MHDA":
|
||||
result.mhda_from = term.text
|
||||
result.mhda_to = term.text
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
else:
|
||||
result.mhda_from = term.text
|
||||
result.mhda_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("MHDA")
|
||||
elif term.field == "LR":
|
||||
result.lr_from = term.text
|
||||
result.lr_to = term.text
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
else:
|
||||
result.lr_from = term.text
|
||||
result.lr_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("LR")
|
||||
elif term.field == "DCOM":
|
||||
result.dcom_from = term.text
|
||||
result.dcom_to = term.text
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
else:
|
||||
result.dcom_from = term.text
|
||||
result.dcom_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("DCOM")
|
||||
elif term.field == "DEP":
|
||||
result.dep_from = term.text
|
||||
result.dep_to = term.text
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
else:
|
||||
result.dep_from = term.text
|
||||
result.dep_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("DEP")
|
||||
elif term.field == "__RANGE_DP__":
|
||||
|
||||
@@ -38,7 +38,7 @@ class AdvancedSearchEngine:
|
||||
tag_ids: list[str] | None,
|
||||
retracted: str, negative_result: str,
|
||||
is_oa: bool | None, language: str | None, languages: list[str] | None, nlm_subsets: list[str] | None,
|
||||
page_size: int, sort: str,
|
||||
page_size: int, sort: str, page: int = 1,
|
||||
# PubMed filter params
|
||||
has_abstract: bool | None = None,
|
||||
is_free_full_text: bool | None = None,
|
||||
@@ -66,7 +66,7 @@ class AdvancedSearchEngine:
|
||||
"r": retracted, "nr": negative_result,
|
||||
"oa": is_oa, "lang": language, "lgs": sorted(languages) if languages else [],
|
||||
"ns": sorted(nlm_subsets) if nlm_subsets else [],
|
||||
"ps": page_size, "s": sort,
|
||||
"ps": page_size, "s": sort, "p": page,
|
||||
"ha": has_abstract,
|
||||
"fft": is_free_full_text,
|
||||
"hft": has_full_text,
|
||||
@@ -185,7 +185,7 @@ class AdvancedSearchEngine:
|
||||
journal_tiers, pub_types, tag_ids,
|
||||
retracted, negative_result,
|
||||
is_oa, language, languages, nlm_subsets,
|
||||
page_size, sort,
|
||||
page_size, sort, page,
|
||||
has_abstract=has_abstract,
|
||||
is_free_full_text=is_free_full_text,
|
||||
has_full_text=has_full_text,
|
||||
@@ -216,8 +216,14 @@ class AdvancedSearchEngine:
|
||||
# 谢缓存:只存了 lit_ids + 聚合数据,需从 ②③ 缓存回填 tags/journal
|
||||
tier_map, name_map = await AdvancedSearchEngine._get_journal_map(db)
|
||||
items = await AdvancedSearchEngine._hydrate_items(db, cached["lit_ids"], tier_map, name_map)
|
||||
# year_counts 从独立 facet 缓存取
|
||||
year_counts = await _cache.get(_facet_cache_key) or []
|
||||
# year_counts 从独立 facet 缓存取(兼容 list 和 dict 两种格式)
|
||||
yc_cached = await _cache.get(_facet_cache_key)
|
||||
if isinstance(yc_cached, dict):
|
||||
year_counts = yc_cached.get("year_counts", [])
|
||||
elif isinstance(yc_cached, list):
|
||||
year_counts = yc_cached
|
||||
else:
|
||||
year_counts = []
|
||||
return {
|
||||
"items": items,
|
||||
"total": cached["total"],
|
||||
@@ -446,13 +452,19 @@ class AdvancedSearchEngine:
|
||||
|
||||
# 撤稿过滤
|
||||
if retracted == "no":
|
||||
conditions.append(GlobalLiterature.retracted == False)
|
||||
conditions.append(or_(
|
||||
GlobalLiterature.retracted == False,
|
||||
GlobalLiterature.retracted.is_(None),
|
||||
))
|
||||
elif retracted in ("only", "yes"):
|
||||
conditions.append(GlobalLiterature.retracted == True)
|
||||
|
||||
# 阴性结果过滤
|
||||
if negative_result == "no":
|
||||
conditions.append(GlobalLiterature.is_negative_result == False)
|
||||
conditions.append(or_(
|
||||
GlobalLiterature.is_negative_result == False,
|
||||
GlobalLiterature.is_negative_result.is_(None),
|
||||
))
|
||||
elif negative_result in ("only", "yes"):
|
||||
conditions.append(GlobalLiterature.is_negative_result == True)
|
||||
|
||||
@@ -483,7 +495,7 @@ class AdvancedSearchEngine:
|
||||
GlobalLiterature.abstract.isnot(None),
|
||||
GlobalLiterature.abstract != '',
|
||||
))
|
||||
if is_free_full_text:
|
||||
if is_free_full_text and is_oa is None:
|
||||
conditions.append(GlobalLiterature.is_oa == True)
|
||||
if has_full_text:
|
||||
conditions.append(GlobalLiterature.pmc_id.isnot(None))
|
||||
@@ -504,7 +516,7 @@ class AdvancedSearchEngine:
|
||||
for ui in sex
|
||||
]))
|
||||
if age:
|
||||
from app.api.v1.features import AGE_GROUP_UI_MAP
|
||||
from app.core.constants import AGE_GROUP_UI_MAP
|
||||
age_uis = []
|
||||
for val in age:
|
||||
if val in AGE_GROUP_UI_MAP:
|
||||
@@ -583,7 +595,7 @@ class AdvancedSearchEngine:
|
||||
except Exception:
|
||||
logger.exception("Year counts query failed")
|
||||
year_counts = []
|
||||
await _cache.set(_facet_cache_key, year_counts, ttl=1800)
|
||||
# 第 1 页结束时统一写 facet 缓存(含 total),此处不重复写入
|
||||
|
||||
# 排序(PubMed 查询时跳过 ts_rank,避免语法标签噪音)
|
||||
_relevance_query = query
|
||||
@@ -591,18 +603,18 @@ class AdvancedSearchEngine:
|
||||
# 用纯文本词做相关性排序,去掉 [field] 标签
|
||||
plain_parts = [t.text for t in _pubmed_parsed.plain_terms]
|
||||
plain_parts += [t.text for t in _pubmed_parsed.title_terms]
|
||||
plain_parts += [t.text for t in _pubmed_parsed.abstract_terms]
|
||||
plain_parts += [t.text for t in _pubmed_parsed.tiab_terms]
|
||||
_relevance_query = " ".join(plain_parts) if plain_parts else ""
|
||||
# ── 通用 keyset 分页(所有列式排序模式统一,代替 OFFSET) ──
|
||||
_keyset_cond = AdvancedSearchEngine._keyset_condition(sort, cursor_val, cursor_id)
|
||||
q = select(GlobalLiterature)
|
||||
if _keyset_cond is not None:
|
||||
conditions.append(_keyset_cond)
|
||||
elif sort not in AdvancedSearchEngine.KEYSET_COLUMN_SORTS and page > 1:
|
||||
# P1-F5: best_match/relevance 不支持 keyset,用 OFFSET 翻页
|
||||
elif page > 1:
|
||||
# P1-F5: keyset 条件不存在时(键集排序但游标无效/非键集排序),用 OFFSET 翻页
|
||||
q = q.offset((page - 1) * page_size)
|
||||
|
||||
# 重新构建查询
|
||||
q = select(GlobalLiterature)
|
||||
if conditions:
|
||||
q = q.where(and_(*conditions))
|
||||
q = q.order_by(*AdvancedSearchEngine._apply_order_by(sort, _relevance_query))
|
||||
@@ -722,6 +734,11 @@ class AdvancedSearchEngine:
|
||||
"journal_tier": tier_map.get(lit.journal_issn),
|
||||
"pub_types": lit.pub_types,
|
||||
"affiliation": authors[0].get("affiliation", "") if authors else "",
|
||||
"study_design": lit.study_design,
|
||||
"trial_reg": lit.trial_reg,
|
||||
"retracted": lit.retracted,
|
||||
"is_negative_result": lit.is_negative_result,
|
||||
"rct_detection": lit.rct_detection,
|
||||
})
|
||||
return results
|
||||
|
||||
@@ -779,13 +796,17 @@ class AdvancedSearchEngine:
|
||||
for fld, terms in field_map.items():
|
||||
if not terms:
|
||||
continue
|
||||
field_conds = []
|
||||
pos_conds = []
|
||||
neg_conds = []
|
||||
for term in terms:
|
||||
cond = AdvancedSearchEngine._field_condition(fld, term.text, term.exact)
|
||||
if term.is_not:
|
||||
cond = not_(cond)
|
||||
field_conds.append(cond)
|
||||
term_conditions.append(field_combine(*field_conds) if len(field_conds) > 1 else field_conds[0])
|
||||
neg_conds.append(not_(cond))
|
||||
else:
|
||||
pos_conds.append(cond)
|
||||
if pos_conds:
|
||||
term_conditions.append(field_combine(*pos_conds) if len(pos_conds) > 1 else pos_conds[0])
|
||||
term_conditions.extend(neg_conds)
|
||||
|
||||
# 2. 纯文本词(无字段标签)— P0-2: 对无标签词补充 ATM MeSH 展开
|
||||
# P0-F1: ATM 只展开肯定词,否定词独立 AND,避免被 ATM OR 短路
|
||||
@@ -1045,7 +1066,9 @@ class AdvancedSearchEngine:
|
||||
for t in subl:
|
||||
val = t.text.upper()
|
||||
if val == "PUBMED":
|
||||
continue # no-op: 所有记录都是 PubMed
|
||||
if is_neg:
|
||||
term_conditions.append(text("FALSE"))
|
||||
continue
|
||||
elif val == "MEDLINE":
|
||||
cond = GlobalLiterature.citation_status == "medline"
|
||||
elif val.isalpha():
|
||||
@@ -1085,23 +1108,39 @@ class AdvancedSearchEngine:
|
||||
# 处理括号分组的词(保留 OR/AND 嵌套结构,P2-2)
|
||||
if pp.groups:
|
||||
for idx, group in enumerate(pp.groups):
|
||||
group_conds = []
|
||||
all_not = all(t.is_not for t in group)
|
||||
g_pos = []
|
||||
g_neg = []
|
||||
for t in group:
|
||||
cond = await AdvancedSearchEngine._single_term_condition(db, t)
|
||||
if cond is not None:
|
||||
if not all_not and t.is_not:
|
||||
cond = not_(cond)
|
||||
group_conds.append(cond)
|
||||
if group_conds:
|
||||
gop = (pp.group_operators[idx]
|
||||
if idx < len(pp.group_operators)
|
||||
else "and")
|
||||
combine_fn = or_ if gop == "or" else and_
|
||||
combined = combine_fn(*group_conds) if len(group_conds) > 1 else group_conds[0]
|
||||
if all_not and len(group_conds) >= 1:
|
||||
combined = not_(combined)
|
||||
term_conditions.append(combined)
|
||||
if all_not:
|
||||
g_neg.append(cond) # raw condition,外部统一 not_()
|
||||
elif t.is_not:
|
||||
g_neg.append(not_(cond))
|
||||
else:
|
||||
g_pos.append(cond)
|
||||
gop = (pp.group_operators[idx]
|
||||
if idx < len(pp.group_operators)
|
||||
else "and")
|
||||
combine_fn = or_ if gop == "or" else and_
|
||||
|
||||
if all_not:
|
||||
# NOT(A OR B): single UnaryExpression → 顶层 OR/NOT 分离时被检测为 neg → 独立 AND
|
||||
if g_neg:
|
||||
combined = combine_fn(*g_neg) if len(g_neg) > 1 else g_neg[0]
|
||||
term_conditions.append(not_(combined))
|
||||
else:
|
||||
# 混合/正组:将 pos 和 neg 按组操作符组合,保留组内结构
|
||||
# 避免 neg 被 OR/NOT 分离拉出来破坏语义
|
||||
combined = None
|
||||
if g_pos:
|
||||
combined = combine_fn(*g_pos) if len(g_pos) > 1 else g_pos[0]
|
||||
if g_neg:
|
||||
neg_combined = combine_fn(*g_neg) if len(g_neg) > 1 else g_neg[0]
|
||||
combined = combine_fn(combined, neg_combined) if combined is not None else neg_combined
|
||||
if combined is not None:
|
||||
term_conditions.append(combined)
|
||||
|
||||
# 将 term_conditions 加入 conditions
|
||||
if term_conditions:
|
||||
@@ -1227,7 +1266,7 @@ class AdvancedSearchEngine:
|
||||
try:
|
||||
return GlobalLiterature.pmid == int(term.text)
|
||||
except ValueError:
|
||||
return None
|
||||
return GlobalLiterature.doi.ilike(f"%{_escape_ilike(term.text)}%")
|
||||
if field == "DOI":
|
||||
return GlobalLiterature.doi.ilike(f"%{_escape_ilike(term.text)}%")
|
||||
if field == "GR":
|
||||
@@ -1262,7 +1301,7 @@ class AdvancedSearchEngine:
|
||||
return text("TRUE") # no-op: 所有记录都是 PubMed
|
||||
elif val == "MEDLINE":
|
||||
return GlobalLiterature.citation_status == "medline"
|
||||
elif len(val) == 1 and val.isalpha():
|
||||
elif val.isalpha():
|
||||
subq = select(GlobalJournal.issn).where(
|
||||
GlobalJournal.nlm_subsets.overlap([val])
|
||||
)
|
||||
@@ -1304,7 +1343,14 @@ class AdvancedSearchEngine:
|
||||
elif field == "abstract":
|
||||
return GlobalLiterature.abstract.ilike(_pt())
|
||||
elif field == "author":
|
||||
return GlobalLiterature.author_names_text.ilike(_pt())
|
||||
pat = _pt()
|
||||
return or_(
|
||||
text(
|
||||
"EXISTS (SELECT 1 FROM jsonb_array_elements(global_literature.authors) AS _a "
|
||||
"WHERE _a->>'family' ILIKE :author_pat)"
|
||||
).bindparams(author_pat=pat),
|
||||
GlobalLiterature.author_names_text.ilike(pat),
|
||||
)
|
||||
elif field == "journal":
|
||||
pat = _pt()
|
||||
return or_(
|
||||
@@ -1342,6 +1388,9 @@ class AdvancedSearchEngine:
|
||||
return or_(
|
||||
GlobalLiterature.title.ilike(pat),
|
||||
GlobalLiterature.abstract.ilike(pat),
|
||||
GlobalLiterature.author_names_text.ilike(pat),
|
||||
GlobalLiterature.journal.ilike(pat),
|
||||
GlobalLiterature.journal_iso.ilike(pat),
|
||||
cast(GlobalLiterature.pmid, String).ilike(pat),
|
||||
GlobalLiterature.doi.ilike(pat),
|
||||
)
|
||||
@@ -1356,14 +1405,25 @@ class AdvancedSearchEngine:
|
||||
GlobalLiterature.title.ilike(like_val),
|
||||
cast(GlobalLiterature.pmid, String).ilike(like_val),
|
||||
GlobalLiterature.doi.ilike(like_val),
|
||||
GlobalLiterature.abstract.ilike(like_val),
|
||||
GlobalLiterature.author_names_text.ilike(like_val),
|
||||
GlobalLiterature.journal.ilike(like_val),
|
||||
)
|
||||
# P7-D2: Chinese → ILIKE fallback (tsvector is English-only)
|
||||
if re.search(r'[一-鿿㐀-䶿豈-]', term):
|
||||
return or_(
|
||||
GlobalLiterature.title.ilike(like_val),
|
||||
GlobalLiterature.abstract.ilike(like_val),
|
||||
GlobalLiterature.author_names_text.ilike(like_val),
|
||||
GlobalLiterature.journal.ilike(like_val),
|
||||
)
|
||||
return GlobalLiterature.search_tsv.op("@@")(func.plainto_tsquery("english", term))
|
||||
# tsvector 索引主覆盖 title/abstract/author_names/chemicals/genes/mesh/keywords
|
||||
# journal/journal_iso/affiliation 不在 tsvector 中,以 ILIKE 兜底
|
||||
return or_(
|
||||
GlobalLiterature.search_tsv.op("@@")(func.plainto_tsquery("english", term)),
|
||||
GlobalLiterature.journal.ilike(like_val),
|
||||
GlobalLiterature.journal_iso.ilike(like_val),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _expand_mesh_tag_ids(
|
||||
@@ -1408,6 +1468,7 @@ class AdvancedSearchEngine:
|
||||
q = m.strip().lower()
|
||||
if not q:
|
||||
continue
|
||||
# entry_terms 在 import_mesh_full.py 时已统一小写,可安全用 @> 精确匹配
|
||||
entry_conds.append(GlobalTag.entry_terms.contains([q]))
|
||||
name_conds.append(GlobalTag.name_en.ilike(_escape_ilike(m)))
|
||||
|
||||
@@ -1516,11 +1577,15 @@ class AdvancedSearchEngine:
|
||||
rank = func.ts_rank(GlobalLiterature.search_tsv, tsq)
|
||||
return [rank.desc(), GlobalLiterature.id.desc()]
|
||||
elif sort == "first_author":
|
||||
return [GlobalLiterature.authors[0]['family'].astext.asc().nullslast()]
|
||||
return [GlobalLiterature.authors[0]['family'].astext.asc().nullslast(),
|
||||
GlobalLiterature.id.asc()]
|
||||
elif sort == "journal":
|
||||
return [GlobalLiterature.journal.asc().nullslast()]
|
||||
return [GlobalLiterature.journal.asc().nullslast(),
|
||||
GlobalLiterature.journal_iso.asc().nullslast(),
|
||||
GlobalLiterature.id.asc()]
|
||||
elif sort == "title":
|
||||
return [GlobalLiterature.title.asc().nullslast()]
|
||||
return [GlobalLiterature.title.asc().nullslast(),
|
||||
GlobalLiterature.id.asc()]
|
||||
else:
|
||||
return [GlobalLiterature.pub_date.desc().nullslast(), GlobalLiterature.id.desc()]
|
||||
|
||||
@@ -1544,34 +1609,50 @@ class AdvancedSearchEngine:
|
||||
return None
|
||||
|
||||
try:
|
||||
if cursor_val == "__NULL__":
|
||||
if sort == "date":
|
||||
return and_(GlobalLiterature.pub_date.is_(None), GlobalLiterature.id < uid)
|
||||
elif sort == "cited":
|
||||
return and_(GlobalLiterature.cited_by_count.is_(None), GlobalLiterature.id < uid)
|
||||
elif sort == "title":
|
||||
return and_(GlobalLiterature.title.is_(None), GlobalLiterature.id > uid)
|
||||
elif sort == "journal":
|
||||
return and_(GlobalLiterature.journal.is_(None), GlobalLiterature.id > uid)
|
||||
elif sort == "first_author":
|
||||
return and_(GlobalLiterature.authors[0]['family'].astext.is_(None), GlobalLiterature.id > uid)
|
||||
if sort == "date":
|
||||
from datetime import date as dt_date
|
||||
val = dt_date.fromisoformat(cursor_val)
|
||||
return or_(
|
||||
GlobalLiterature.pub_date < val,
|
||||
and_(GlobalLiterature.pub_date == val, GlobalLiterature.id < uid),
|
||||
and_(GlobalLiterature.pub_date.is_(None), GlobalLiterature.id < uid),
|
||||
)
|
||||
elif sort == "cited":
|
||||
val = int(cursor_val)
|
||||
return or_(
|
||||
GlobalLiterature.cited_by_count < val,
|
||||
and_(GlobalLiterature.cited_by_count == val, GlobalLiterature.id < uid),
|
||||
and_(GlobalLiterature.cited_by_count.is_(None), GlobalLiterature.id < uid),
|
||||
)
|
||||
elif sort == "title":
|
||||
return or_(
|
||||
GlobalLiterature.title > cursor_val,
|
||||
and_(GlobalLiterature.title == cursor_val, GlobalLiterature.id > uid),
|
||||
and_(GlobalLiterature.title.is_(None), GlobalLiterature.id > uid),
|
||||
)
|
||||
elif sort == "journal":
|
||||
return or_(
|
||||
GlobalLiterature.journal > cursor_val,
|
||||
and_(GlobalLiterature.journal == cursor_val, GlobalLiterature.id > uid),
|
||||
and_(GlobalLiterature.journal.is_(None), GlobalLiterature.id > uid),
|
||||
)
|
||||
elif sort == "first_author":
|
||||
family_col = GlobalLiterature.authors[0]['family'].astext
|
||||
return or_(
|
||||
family_col > cursor_val,
|
||||
and_(family_col == cursor_val, GlobalLiterature.id > uid),
|
||||
and_(family_col.is_(None), GlobalLiterature.id > uid),
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
@@ -1582,14 +1663,16 @@ class AdvancedSearchEngine:
|
||||
"""从末尾条目标提取 keyset 游标值。"""
|
||||
if sort == "date":
|
||||
val = str(lit.pub_date or lit.article_date or "")
|
||||
return val if val else None
|
||||
return val if val else "__NULL__"
|
||||
elif sort == "cited":
|
||||
return str(lit.cited_by_count) if lit.cited_by_count is not None else None
|
||||
if lit.cited_by_count is not None:
|
||||
return str(lit.cited_by_count)
|
||||
return "__NULL__"
|
||||
elif sort == "title":
|
||||
return lit.title or None
|
||||
return lit.title or "__NULL__"
|
||||
elif sort == "journal":
|
||||
return lit.journal or None
|
||||
return lit.journal or "__NULL__"
|
||||
elif sort == "first_author":
|
||||
authors = lit.authors or []
|
||||
return authors[0].get("family") if authors else None
|
||||
return authors[0].get("family") if authors else "__NULL__"
|
||||
return None
|
||||
|
||||
@@ -25,6 +25,9 @@ async def shutdown(ctx):
|
||||
async def daily_ftp_update(ctx):
|
||||
"""每日 FTP 增量更新(取代旧的精搜+宽搜+retagger)"""
|
||||
stats = await run_daily_ftp_update(ctx)
|
||||
# 搜索缓存失效:管道新增/修改文献后,缓存中的搜索结果立即过时
|
||||
from app.core.cache import cache
|
||||
await cache.invalidate_search_cache()
|
||||
return stats if isinstance(stats, dict) else {"status": "ok"}
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ pubmed_filter:
|
||||
mesh_cross_include:
|
||||
- "E02.319" # Antineoplastic agents
|
||||
- "E02.815" # Radiotherapy
|
||||
- "D27.505" # Antineoplastic drugs
|
||||
- "D27.505.954.248" # Antineoplastic agents (D27 subclass)
|
||||
pub_type_prefer:
|
||||
- "Guideline"
|
||||
- "Randomized Controlled Trial"
|
||||
|
||||
@@ -224,6 +224,25 @@ async def apply():
|
||||
merged += 1
|
||||
print(f" 合并: {tag.name_zh}/{tag.name_en} ← {dup_tag.name_en} (移动 {moved} 篇)")
|
||||
|
||||
# 更新 tag_ids 数组:移除已删除标签的 ID,确保种子标签 ID 存在
|
||||
from sqlalchemy import text as _sa_text
|
||||
await db.execute(
|
||||
_sa_text("""
|
||||
UPDATE global_literature
|
||||
SET tag_ids = ARRAY(
|
||||
SELECT DISTINCT unnest(
|
||||
array_append(
|
||||
array_remove(tag_ids, :dup_id::uuid),
|
||||
:seed_id::uuid
|
||||
)
|
||||
)
|
||||
)
|
||||
WHERE tag_ids @> ARRAY[:dup_id::uuid]
|
||||
"""),
|
||||
{"dup_id": str(dup_tag.id), "seed_id": str(tag.id)}
|
||||
)
|
||||
print(f" tag_ids 数组已同步")
|
||||
|
||||
await db.commit()
|
||||
print(f"\n完成! 合并 {merged} 个重复标签")
|
||||
|
||||
|
||||
@@ -330,27 +330,32 @@ def _extract_article(elem) -> dict | None:
|
||||
|
||||
|
||||
def _is_oncology(article: dict) -> bool:
|
||||
"""判断一篇文献是否属于肿瘤学范畴。
|
||||
"""判断一篇文献是否应收录。
|
||||
|
||||
两层过滤:
|
||||
Tier 1 — 有 MeSH 标引 → 仅检查 C04 树号(精确),不检查文本
|
||||
此规则基于数据分析:23.6 万条现有非 C04 误判文献中,
|
||||
有 MeSH 再检查文本会导致大量假阳性。
|
||||
三层过滤:
|
||||
Step 0 — 排除出版类型:
|
||||
Editorial, Letter, News, Comment → 不收录
|
||||
Tier 1 — 有 MeSH 标引 → 检查 C04 树号
|
||||
交叉引入树号(E02.319 抗癌药、E02.815 放疗、D27.505.954.248 抗肿瘤药)
|
||||
不检查文本(避免假阳性)
|
||||
Tier 2 — 无 MeSH 标引 → 关键词评分:
|
||||
- 标题命中任一关键词 → 收录(强信号)
|
||||
- 摘要 ≥2 关键词命中 → 收录
|
||||
- 摘要 0-1 关键词 → 放过(等待标引后回查)
|
||||
- 标题命中任一关键词 → 收录(强信号)
|
||||
- 摘要 ≥2 关键词命中 → 收录
|
||||
- 摘要 0-1 关键词 → 放过(等待标引后回查)
|
||||
"""
|
||||
mf = get_mesh_filter()
|
||||
mesh_headings = article.get('mesh_headings', [])
|
||||
# Step 0: 排除出版类型
|
||||
EXCLUDED_TYPES = {"Editorial", "Letter", "News", "Comment"}
|
||||
if any(pt in EXCLUDED_TYPES for pt in article.get("pub_types", [])):
|
||||
return False
|
||||
|
||||
mesh_headings = article.get("mesh_headings", [])
|
||||
if mesh_headings:
|
||||
# Tier 1: 有 MeSH → 只看 C04 树号
|
||||
return mf.is_oncology_by_mesh(mesh_headings)
|
||||
# Tier 1: 有 MeSH → C04 树号 + 交叉引入
|
||||
return get_mesh_filter().is_oncology_by_mesh(mesh_headings)
|
||||
|
||||
# Tier 2: 无 MeSH → 关键词评分模式
|
||||
return mf.is_oncology_by_text_score(
|
||||
article.get('title'), article.get('abstract')
|
||||
# Tier 2: 无 MeSH → 关键词评分
|
||||
return get_mesh_filter().is_oncology_by_text_score(
|
||||
article.get("title"), article.get("abstract")
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@
|
||||
|
||||
| ID | 项目 | 严重度 | 详情 |
|
||||
|----|------|--------|------|
|
||||
| **P0-4** | tsvector 扩展 | **中** | 迁移 `g0h1i2j3k4l5` 已创建但生产未 apply。添加了 mesh_headings + keywords(weight C)到 tsvector。本地 SQLite 不执行此迁移。生产需手动 `alembic upgrade head` |
|
||||
| **P0-4** | tsvector 扩展 | **中** | 迁移 `g0h1i2j3k4l5` 已创建但生产未 apply。添加了 mesh_headings + keywords(weight C)到 tsvector。需手动 `alembic upgrade head` |
|
||||
| **P1-6** | Affiliation [AD] | **低** | 仍用 `cast(authors, String).ilike()`。JSONB 结构文本(键名)可能产生假阳性。完整修复需 schema 变更 + 迁移。目前实际影响极小(搜索医院名/机构名极少与 JSONB 键名冲突) |
|
||||
| **P2-3** | 布尔优先级 | **低** | 递归下降 parser 正确解析 `A OR B AND C` = `A OR (B AND C)`。但扁平 term_conditions 列表丢失嵌套结构。没有 PubMed 的复杂布尔优先级测试失败案例,仅理论不足 |
|
||||
| **P2-4** | 精确短语非 "all" | **极低** | 对非 "all" 字段,exact=True 与 =False 生成相同 ILIKE。但这是功能正确的——ILIKE 本身不做词干化。有意为之,不影响结果 |
|
||||
|
||||
@@ -69,7 +69,7 @@ class TestGetQuota:
|
||||
mock_session.execute.return_value = mock_result
|
||||
|
||||
q = await mw._get_quota("t1")
|
||||
assert q == 1000 # Pro plan
|
||||
assert q == 10000 # Pro plan
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_db_exception_fallback(self):
|
||||
|
||||
@@ -68,6 +68,20 @@ class TestMeshByMesh:
|
||||
]
|
||||
assert mf.is_oncology_by_mesh(headings) is True
|
||||
|
||||
def test_cross_include_antineoplastic(self, mf):
|
||||
"""交叉引入:Antineoplastic Agents (D27.505.954.248) → True"""
|
||||
assert mf.is_oncology_by_mesh([{"descriptor": "Antineoplastic Agents"}]) is True
|
||||
|
||||
def test_cross_include_radiotherapy(self, mf):
|
||||
"""交叉引入:Radiotherapy (E02.815) → True"""
|
||||
assert mf.is_oncology_by_mesh([{"descriptor": "Radiotherapy"}]) is True
|
||||
|
||||
def test_cross_include_not_oncology(self, mf):
|
||||
"""非肿瘤交叉引入 → False(Antibiotics D27.505.954.122 下有但非 E02.319/D27.505.954.248 下位)"""
|
||||
# Antibiotics 在 D27.505.954.122 下位但不在抗肿瘤范畴,由具体子类决定
|
||||
# 用明确非肿瘤的 MeSH 验证
|
||||
assert mf.is_oncology_by_mesh([{"descriptor": "Anti-Bacterial Agents"}]) is False
|
||||
|
||||
|
||||
class TestMeshByText:
|
||||
def test_cancer_in_title(self):
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ specialty:
|
||||
pubmed_filter:
|
||||
mesh_include_categories: ["C04"]
|
||||
mesh_include_subcategories: ["C04.557", "C04.588", "C04.697"]
|
||||
mesh_cross_include: ["E02.319", "E02.815", "D27.505"]
|
||||
mesh_cross_include: ["E02.319", "E02.815", "D27.505.954.248"]
|
||||
|
||||
journals:
|
||||
tier_config:
|
||||
|
||||
+29
-9
@@ -118,7 +118,7 @@ EDAT 对应字段:`GlobalLiterature.entrez_date`(DATE,新增字段,迁
|
||||
pubmed_filter:
|
||||
mesh_include_categories: ["C04"]
|
||||
mesh_include_subcategories: ["C04.557", "C04.588", "C04.697"]
|
||||
mesh_cross_include: ["E02.319", "E02.815", "D27.505"]
|
||||
mesh_cross_include: ["E02.319", "E02.815", "D27.505.954.248"]
|
||||
pub_type_prefer:
|
||||
- "Guideline"
|
||||
- "Randomized Controlled Trial"
|
||||
@@ -135,6 +135,16 @@ pubmed_filter:
|
||||
- "Comment"
|
||||
```
|
||||
|
||||
**三层过滤逻辑**(`_is_oncology()` 实现):
|
||||
|
||||
| 层级 | 条件 | 动作 |
|
||||
|------|------|------|
|
||||
| **Step 0** `pub_type_exclude` | PublicationType ∈ {Editorial, Letter, News, Comment} | ❌ 直接丢弃 |
|
||||
| **Step 1** MeSH 树号匹配 | MeSH heading 在 C04(Neoplasms)或 cross-include 树号(E02.319 抗癌药、E02.815 放疗、D27.505.954.248 抗肿瘤药)下 | ✅ 收录 |
|
||||
| **Step 2** 文本评分兜底 | 无 MeSH 标引的文献:标题含关键词 → 收录;摘要含 ≥2 个关键词 → 收录;摘要单关键词 → 放过 | ✅/❌ 按评分 |
|
||||
|
||||
三层按顺序执行:Step 0 最先(直接排除),Step 1 中间(MeSH 精确匹配),Step 2 最后(无 MeSH 时的文本兜底)。
|
||||
|
||||
---
|
||||
|
||||
## 二、Pipeline 完整流程
|
||||
@@ -194,15 +204,25 @@ ARQ 定时任务 (每日 FTP 更新 03:07 UTC)
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ Step 3: 肿瘤科过滤(配置驱动,复用 _is_oncology()) │
|
||||
│ Step 3: 肿瘤科过滤(三层,配置驱动,复用 _is_oncology()) │
|
||||
│ │
|
||||
│ 对每篇文章的 MeSH Headings[] 逐一检查: │
|
||||
│ if 任一 heading.ui 在 mesh_include_categories 中 │
|
||||
│ → 保留 │
|
||||
│ elif 任一 heading.ui 在 mesh_cross_include 中 │
|
||||
│ → 保留(交叉领域) │
|
||||
│ else │
|
||||
│ → 丢弃 │
|
||||
│ 对每篇文章按三层逐一判断: │
|
||||
│ │
|
||||
│ Step 0 — 排除类型检查: │
|
||||
│ if publication_type ∈ {Editorial, Letter, News, │
|
||||
│ Comment} │
|
||||
│ → ❌ 丢弃(无需往下判断) │
|
||||
│ │
|
||||
│ Step 1 — MeSH 树号匹配: │
|
||||
│ elif 任一 heading 在 C04(Neoplasms)下 → ✅ 收录 │
|
||||
│ elif 任一 heading 在 cross-include 树号下 │
|
||||
│ (E02.319 抗癌药 / E02.815 放疗 /│
|
||||
│ D27.505.954.248 抗肿瘤药) → ✅ 收录 │
|
||||
│ │
|
||||
│ Step 2 — 文本评分兜底(无 MeSH 标引时): │
|
||||
│ elif 标题含关键词 → ✅ 收录 │
|
||||
│ elif 摘要含 ≥2 个关键词 → ✅ 收录 │
|
||||
│ else → ❌ 丢弃 │
|
||||
│ │
|
||||
│ 过滤结果:日均 ~5000 条总记录 → 约 400-800 条肿瘤相关 │
|
||||
│ 预计耗时:1-2 分钟 │
|
||||
|
||||
+2
-2
@@ -26,7 +26,7 @@ Phase 1 (5天) ✓ Phase 2 (8天) ✓ Phase 3 (8天)
|
||||
Phase 4 (新增 — 完成) Phase 5 (新增 — 完成) Phase 6 (新增 — 完成)
|
||||
Meta 分析基础设施 搜索增强 + 数据深化 数据 + AI 深化
|
||||
|
||||
├─ 系统评价 CRUD ├─ PMC OA 全文管道 ├─ SQLite → PG 迁移
|
||||
├─ 系统评价 CRUD ├─ PMC OA 全文管道 ├─ PG 全文搜索迁移
|
||||
├─ 研究设计分类引擎 ├─ Table 1 结构化提取 ├─ tsvector 全文搜索
|
||||
├─ RCT 两档检测(confirmed/susp) ├─ 许可证/撤稿标记 ├─ 纳入篇数额度模型
|
||||
├─ PICO AI 抽取 ├─ 阴性结果检测 ├─ 机构版私有化交付
|
||||
@@ -198,7 +198,7 @@ Phase 4 ──→ Phase 5 ──→ Phase 6
|
||||
| 4. tsvector 迁移脚本 | Alembic: `e8f9a0b1c2d3_add_search_tsv` 已存在 | - | ✅ |
|
||||
| 5. tsvector 触发器 | BEFORE INSERT OR UPDATE 触发器 + backfill 已存在 | 步骤 4 | ✅ |
|
||||
| 6. 重写搜索 | `search_engine.py` ILIKE → `search_tsv @@ plainto_tsquery` + `ts_rank` 排序 | 步骤 5 | ✅ |
|
||||
| 7. 删除 SQLite 兼容垫片 | `study_design` 改用 JSON path;`pub_types` JSONB cast PG 原生 | 步骤 6 | ✅ |
|
||||
| 7. 数据库适配层清理 | `study_design` 改用 JSON path;`pub_types` JSONB cast PG 原生 | 步骤 6 | ✅ |
|
||||
| 8. 机构版交付文档 | docs/deployment-guide.md + docker-compose.prod.yml 说明 | 步骤 7 | ✅ |
|
||||
| 9. Europe PMC 游标分页 | `pubmed_api.py` + `admin.py` — cursor-based search,`?_source=europe_pmc` | 步骤 8 | ✅ |
|
||||
| 10. DOI 跨源去重 | `_process_article` DOI 匹配 → 同 DOI 不同 PMID 时原地更新 | 步骤 9 | ✅ |
|
||||
|
||||
+2
-2
@@ -1,9 +1,9 @@
|
||||
# 搜索功能实施计划
|
||||
|
||||
> 计划日期:2026-07-24
|
||||
> **最后更新**:2026-07-27(四轮修复已完成)
|
||||
> **最后更新**:2026-07-28(八轮修复已完成)
|
||||
> **硬性目标:搜索功能必须与 PubMed 完全一致。不允许"暂缓/可以忽略/不急"的降级。**
|
||||
> **当前状态:42+ 字段标签已接通,127 项搜索测试通过,核心功能就绪。**
|
||||
> **当前状态:1007 项测试全部通过,前端构建成功,搜索缓存失效、Redis 重试、中文标签、翻页稳定性等全面加固。**
|
||||
> 基于 9 Agent 审计 + 真实数据库 1,662 篇字段覆盖率验证
|
||||
|
||||
---
|
||||
|
||||
+89
-5
@@ -2,8 +2,8 @@
|
||||
|
||||
> 本文档按修复轮次详细记录所有搜索功能合规性修复的背景、根因分析和修改内容。
|
||||
>
|
||||
> **累计**:7 轮,98 项修复,50+ 字段标签注册,276 项测试覆盖,7 项已知限制
|
||||
> **时间跨度**:2026-07-24 ~ 2026-07-27
|
||||
> **累计**:8 轮,110 项修复,50+ 字段标签注册,1007 项测试覆盖,7 项已知限制
|
||||
> **时间跨度**:2026-07-24 ~ 2026-07-28
|
||||
> **核心文件**:`pubmed_query_parser.py`(~730 行)→ `search_engine.py`(~1320 行)
|
||||
|
||||
---
|
||||
@@ -16,7 +16,9 @@
|
||||
4. [第四轮:字段补全与语义优化(11 项)](#第四轮字段补全与语义优化)
|
||||
5. [第五轮:第 5 轮全面审计修复(4 项)](#第五轮第-5-轮全面审计修复)
|
||||
6. [第六轮:第 6 轮全面审计修复(12 项)](#第六轮第-6-轮全面审计修复)
|
||||
7. [遗留限制](#遗留限制)
|
||||
7. [第七轮:第 7 轮深度审计修复(20 项)](#第七轮第-7-轮深度审计修复)
|
||||
8. [第八轮:第 8 轮深度审计修复(12 项)](#第八轮第-8-轮深度审计修复)
|
||||
9. [遗留限制](#遗留限制)
|
||||
|
||||
---
|
||||
|
||||
@@ -642,9 +644,91 @@
|
||||
|
||||
---
|
||||
|
||||
## 第八轮:第 8 轮深度审计修复(12 项)
|
||||
|
||||
**日期**:2026-07-28
|
||||
**数量**:12 项(3 Agent 最新深度审计)
|
||||
**触发**:用户第 4/5 次要求全面检查
|
||||
**测试**:1007 全部通过,前端构建成功
|
||||
|
||||
### P8-1: ATM 缓存未失效(CRITICAL)
|
||||
|
||||
- **文件**:`cache.py:161-168`
|
||||
- **修复**:`invalidate_search_cache()` 新增 `await self.delete_pattern("atm:*")` 清理 MeSH 自动词表映射缓存
|
||||
- **根因**:管道运行后 `atm:*` 缓存保留,新 MeSH 标签在一小时内不被发现
|
||||
|
||||
### P8-2: Pro 方案配额低于 Free(CRITICAL)
|
||||
|
||||
- **文件**:`plans.py:37`
|
||||
- **修复**:`api_quota_per_day: 1_000` → `10_000`
|
||||
- **影响**:Pro 用户不再比 Free 用户更受限
|
||||
|
||||
### P8-3: Redis 连接失败永不重试(CRITICAL)
|
||||
|
||||
- **文件**:`cache.py:20-36`、`rate_limiter.py:37-49`
|
||||
- **修复**:`redis_failed` 标记 60 秒后自动复位,两处统一添加 `_redis_retry_at` + 60s 退避
|
||||
|
||||
### P8-4: 普通搜索中文标签匹配缺失(CRITICAL)
|
||||
|
||||
- **文件**:`literature.py:276-290`
|
||||
- **修复**:检测中文输入后查询 `GlobalTag.name_zh`,匹配时注入 `GlobalLiterature.id IN (子查询)` 标签条件
|
||||
- **根因**:普通搜索只做 tsvector + title/abstract ILIKE,完全绕过 MeSH 标签。中文"肺癌"在英文 tsvector 中命中率极低
|
||||
|
||||
### P8-5: 限速器突发窗口内存泄漏(HIGH)
|
||||
|
||||
- **文件**:`rate_limiter.py`
|
||||
- **修复**:添加 `_cleanup_stale_burst_windows()` 每 500 次请求清理过期 key,添加 `_burst_cleanup_counter`
|
||||
- **影响**:每唯一 IP 在 `_burst_windows` 留下条目,生产环境数千 IP 可能累积
|
||||
|
||||
### P8-6: ASC 排序缺 id tiebreaker(MEDIUM)
|
||||
|
||||
- **文件**:`search_engine.py:1568-1574`
|
||||
- **修复**:title/journal/first_author 排序追加 `GlobalLiterature.id.asc()` 作为次级排序列
|
||||
- **根因**:`_keyset_condition` 假设 id 是 tiebreaker(`AND id > uid`),但 `_apply_order_by` 未在 ORDER BY 中包含 id → 值相同的行在翻页时非确定排序
|
||||
|
||||
### P8-7: keyset NULL 值缺 id tiebreaker(MEDIUM)
|
||||
|
||||
- **文件**:`search_engine.py:1595-1631`
|
||||
- **修复**:所有 `is_(None)` 子句添加 `and_(col.is_(None), GlobalLiterature.id < uid / > uid)`
|
||||
- **修复 2**:`_cursor_from_item` 对 NULL 列返回 `"__NULL__"` 哨兵值 → `_keyset_condition` 新增 `__NULL__` 精准处理分支
|
||||
- **根因**:当游标落在 NULL 行后,`is_(None)` 无条件返回所有 NULL 行→重复
|
||||
|
||||
### P8-8: `_field_condition("all")` 缺 journal/affiliation 兜底(MEDIUM)
|
||||
|
||||
- **文件**:`search_engine.py:1381-1415`
|
||||
- **修复**:
|
||||
- tsvector 默认路径追加 `or_(journal ILIKE, journal_iso ILIKE)`
|
||||
- `"/"` 路径追加 abstract、author_names_text、journal
|
||||
- 中文 ILIKE 路径追加 author_names_text、journal
|
||||
- **根因**:tsvector 不含 journal/journal_iso/affiliation,纯文本搜索可能漏 journal 匹配
|
||||
|
||||
### P8-9: Cron 任务缺搜索缓存失效(HIGH)
|
||||
|
||||
- **文件**:`worker.py:25-28`
|
||||
- **修复**:`daily_ftp_update()` 末尾调用 `cache.invalidate_search_cache()`
|
||||
- **根因**:`POST /admin/pipeline/run` 做了缓存失效,但 ARQ 定时任务 `daily_ftp_update`(03:07 UTC)没做→自动 pipeline 后缓存 5-30 分钟过期
|
||||
|
||||
### P8-10: AdvancedPubSearchView 交替 #N 循环引用(CRITICAL)
|
||||
|
||||
- **文件**:`frontend/.../AdvancedPubSearchView.vue:221-239`
|
||||
- **修复**:`resolveQuery()` 添加 `seen Set<string>` 检测交替循环(#1→#2→#1)
|
||||
- **根因**:原循环检测只检查 `current === prev`,对交替引用无效→10 轮迭代产生嵌套垃圾
|
||||
|
||||
### P8-11: URL date_preset/year_from/year_to 冲突(MEDIUM)
|
||||
|
||||
- **文件**:`frontend/.../SearchView.vue:380-396`
|
||||
- **修复**:`restoreFromQuery` 中 `date_preset` 优先;date_preset 存在时清空 year_from/year_to;无 date_preset 且无 date_from/date_to 时再读 year_from/year_to
|
||||
|
||||
### P8-12: 增加中文搜索路径(enhancement)
|
||||
|
||||
- **文件**:`search_engine.py:1397-1414`
|
||||
- **修复**:`_field_condition("all")` 的 `/` 路径和中文路径补充 author_names_text、journal ILIKE 覆盖
|
||||
|
||||
---
|
||||
|
||||
## 遗留限制
|
||||
|
||||
截至 2026-07-27,剩余 7 项已知限制:
|
||||
截至 2026-07-28,剩余 7 项已知限制:
|
||||
|
||||
| ID | 问题 | 原因 | 影响 |
|
||||
|----|------|------|------|
|
||||
@@ -667,6 +751,6 @@
|
||||
| `test_pubmed_search_integration.py` | ~60 | 字段映射、API 集成、前端格式 |
|
||||
| `test_comprehensive_verify.py` | ~27 | 字段完整、NOT 语义、括号、日期 |
|
||||
| `test_comprehensive_verify.py` | ~55 | 第 7 轮新增覆盖(full dispatch、boolean_operator、cursor 等) |
|
||||
| **合计** | **276** | 全部通过 |
|
||||
| 全量测试套件 | **1007** | 全部通过(含前 7 轮 276 项搜索专项 + 731 项通用测试) |
|
||||
|
||||
> **预存失败(13 项)**:9 项 `feed_engine` `StopAsyncIteration`(测试数据缺失) + 4 项 `pubmed_api` `_tag_article` import(函数已移入 pipeline)
|
||||
|
||||
@@ -5,6 +5,12 @@ interface UsePaginationOptions {
|
||||
fetchFn: (page: number) => Promise<void>
|
||||
}
|
||||
|
||||
/**
|
||||
* Offset-based pagination composable.
|
||||
*
|
||||
* 注意:keyset 分页模式(date/cited/title/journal/first_author 排序)由 SearchView
|
||||
* 独立管理 keysetCursors/keysetHasMore/keysetPage。此 composable 的 totalPages/hasMore
|
||||
* 在 keyset 模式下语义不准确(keyset 用 has_more 标志而非总页数判断),由调用方覆盖使用。 */
|
||||
export function usePagination(opts: UsePaginationOptions) {
|
||||
const { fetchFn, pageSize: initialPageSize = 20 } = opts
|
||||
const page = ref(1)
|
||||
|
||||
@@ -32,18 +32,35 @@ export function resolveQuery(query: string, entries: HistoryEntry[]): string {
|
||||
})
|
||||
}
|
||||
|
||||
/** 递归展开所有 #N 引用为纯查询 */
|
||||
/** 递归展开所有 #N 引用为纯查询,带循环引用检测 */
|
||||
export function expandQuery(query: string, entries: HistoryEntry[]): string {
|
||||
const seen = new Set<string>()
|
||||
let prev = ''
|
||||
let current = query
|
||||
for (let i = 0; i < 10; i++) {
|
||||
if (current === prev) break
|
||||
const refs = current.match(/#(\d+)/g)
|
||||
if (refs) {
|
||||
for (const ref of refs) {
|
||||
if (seen.has(ref)) return prev // 循环引用 → 返回上次安全结果
|
||||
seen.add(ref)
|
||||
}
|
||||
}
|
||||
prev = current
|
||||
current = resolveQuery(current, entries)
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
/** 重新计算所有条目的 expanded_query,在删除/淘汰后保证一致性 */
|
||||
function recomputeExpanded(entries: HistoryEntry[]) {
|
||||
const ids = new Set(entries.map(e => e.id))
|
||||
for (const entry of entries) {
|
||||
entry.expanded_query = expandQuery(entry.query, entries)
|
||||
.replace(/#(\d+)/g, (m) => ids.has(m) ? m : '(deleted)')
|
||||
}
|
||||
}
|
||||
|
||||
export function useSearchHistory() {
|
||||
const entries = ref<HistoryEntry[]>(loadAll())
|
||||
|
||||
@@ -69,6 +86,7 @@ export function useSearchHistory() {
|
||||
if (all.length >= MAX_ENTRIES) {
|
||||
all.sort((a, b) => a.timestamp.localeCompare(b.timestamp))
|
||||
all.shift()
|
||||
recomputeExpanded(all) // 清除悬空 #N 引用
|
||||
}
|
||||
|
||||
all.push(entry)
|
||||
@@ -79,6 +97,7 @@ export function useSearchHistory() {
|
||||
|
||||
function remove(id: string) {
|
||||
const all = loadAll().filter(e => e.id !== id)
|
||||
recomputeExpanded(all) // 重新展开,清除悬空 #N 引用
|
||||
saveAll(all)
|
||||
entries.value = all
|
||||
}
|
||||
@@ -90,8 +109,10 @@ export function useSearchHistory() {
|
||||
|
||||
function download() {
|
||||
const all = loadAll()
|
||||
/** 转义 TSV 特殊字符:制表符/换行/回车 → 空格 */
|
||||
const escapeTsv = (s: string) => s.replace(/[\t\n\r]/g, ' ')
|
||||
const lines = all.map(e =>
|
||||
`${e.id}\t${e.result_count ?? ''}\t${e.timestamp}\t${e.query}`
|
||||
`${e.id}\t${e.result_count ?? ''}\t${e.timestamp}\t${escapeTsv(e.query)}`
|
||||
)
|
||||
const blob = new Blob([lines.join('\n')], { type: 'text/plain;charset=utf-8' })
|
||||
const url = URL.createObjectURL(blob)
|
||||
@@ -99,7 +120,8 @@ export function useSearchHistory() {
|
||||
a.href = url
|
||||
a.download = `pubmed-search-history-${new Date().toISOString().slice(0, 10)}.tsv`
|
||||
a.click()
|
||||
URL.revokeObjectURL(url)
|
||||
// 延迟回收 blob URL,确保浏览器已开始下载
|
||||
setTimeout(() => URL.revokeObjectURL(url), 1000)
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -166,6 +166,7 @@ export interface SupplMeshEntry {
|
||||
|
||||
/** 文献对象(列表用) */
|
||||
export interface LiteratureItem {
|
||||
id?: string
|
||||
pmid: number
|
||||
title?: string
|
||||
doi?: string
|
||||
@@ -327,6 +328,8 @@ export interface SavedFilter {
|
||||
export interface SearchRequestBody {
|
||||
query: string
|
||||
field?: string
|
||||
boolean?: string // "and" | "or"
|
||||
exact_phrase?: boolean
|
||||
page: number
|
||||
page_size: number
|
||||
sort?: string
|
||||
|
||||
@@ -19,6 +19,7 @@ const router = useRouter(); const route = useRoute()
|
||||
const toast = useToast()
|
||||
const auth = useAuthStore()
|
||||
const displaySettings = useDisplaySettings()
|
||||
const KEYSET_SORTS = new Set(['date', 'cited', 'title', 'journal', 'first_author'])
|
||||
|
||||
// ── 搜索参数 ──
|
||||
const query = ref('')
|
||||
@@ -276,6 +277,7 @@ const { page, total, goToPage } = usePagination({
|
||||
}
|
||||
const body: SearchRequestBody = {
|
||||
query: query.value, page: p, page_size: pageSize.value, sort: sort.value,
|
||||
boolean: 'and',
|
||||
}
|
||||
if (field.value !== 'all') body.field = field.value
|
||||
// P3-6: precision_mode 不再发送(后端已忽略)
|
||||
@@ -339,6 +341,7 @@ const { page, total, goToPage } = usePagination({
|
||||
if (retracted.value) body.retracted = retracted.value
|
||||
if (negativeResult.value) body.negative_result = negativeResult.value
|
||||
const { data } = await api.post('/features/search/advanced', body, { signal })
|
||||
if (gen !== searchGeneration.value) return // 丢弃老旧请求
|
||||
results.value = data.items || []
|
||||
// 游标分页:首页走 COUNT 存 total,后续页沿用;cursor 和数据一起返回
|
||||
if (p === 1) total.value = data.total || 0
|
||||
@@ -370,12 +373,14 @@ const { page, total, goToPage } = usePagination({
|
||||
/** 从路由 query 恢复搜索参数 */
|
||||
function restoreFromQuery() {
|
||||
if (route.query.q) query.value = String(route.query.q)
|
||||
if (route.query.field) field.value = String(route.query.field)
|
||||
if (route.query.sort) sort.value = String(route.query.sort)
|
||||
if (route.query.year_from) yearFromStr.value = String(route.query.year_from)
|
||||
if (route.query.year_to) yearToStr.value = String(route.query.year_to)
|
||||
const VALID_FIELDS = new Set(['all', 'title', 'abstract', 'author', 'affiliation', 'journal'])
|
||||
const VALID_SORTS = new Set(['date', 'cited', 'best_match', 'relevance', 'first_author', 'journal', 'title'])
|
||||
if (route.query.field && VALID_FIELDS.has(String(route.query.field))) field.value = String(route.query.field)
|
||||
if (route.query.sort && VALID_SORTS.has(String(route.query.sort))) sort.value = String(route.query.sort)
|
||||
if (route.query.date_preset && ['1y','5y','10y','custom'].includes(String(route.query.date_preset))) {
|
||||
datePreset.value = String(route.query.date_preset)
|
||||
yearFromStr.value = ''
|
||||
yearToStr.value = ''
|
||||
} else if (route.query.date_from || route.query.date_to) {
|
||||
datePreset.value = null
|
||||
if (route.query.date_from) {
|
||||
@@ -388,6 +393,9 @@ function restoreFromQuery() {
|
||||
urlDateTo.value = dt
|
||||
if (dt.length >= 4) yearToStr.value = dt.slice(0, 4)
|
||||
}
|
||||
} else {
|
||||
if (route.query.year_from) yearFromStr.value = String(route.query.year_from)
|
||||
if (route.query.year_to) yearToStr.value = String(route.query.year_to)
|
||||
}
|
||||
if (route.query.tag) selectedTags.value = String(route.query.tag).split(',')
|
||||
if (route.query.tier) selectedTiers.value = String(route.query.tier).split(',')
|
||||
@@ -410,8 +418,8 @@ function restoreFromQuery() {
|
||||
const ps = parseInt(String(route.query.page_size))
|
||||
if (ps >= 10 && ps <= 100) pageSize.value = ps
|
||||
}
|
||||
// keyset 游标不能从 URL 恢复 → page>1 回退到首页(所有排序模式)
|
||||
if (restoredPage.value > 1) restoredPage.value = 1
|
||||
// keyset 游标不能从 URL 恢复 → page>1 回退首页(仅键集排序模式)
|
||||
if (KEYSET_SORTS.has(sort.value) && restoredPage.value > 1) restoredPage.value = 1
|
||||
}
|
||||
|
||||
// 选择预设(1y/5y/10y)时清除自定义年份并自动搜索
|
||||
@@ -507,8 +515,8 @@ function syncSearchToUrl() {
|
||||
if (medlineOnly.value) q.medline_only = 'true'
|
||||
if (excludePreprints.value) q.exclude_preprints = 'true'
|
||||
if (pageSize.value !== 20) q.page_size = String(pageSize.value)
|
||||
// 页码持久化到 URL(不同排序下使用各自的分页)
|
||||
const currentPage = sort.value === 'date' ? keysetPage.value : page.value
|
||||
// 页码持久化到 URL(Keyset 排序使用 keyset 页码,offset 排序使用 offset 页码)
|
||||
const currentPage = KEYSET_SORTS.has(sort.value) ? keysetPage.value : page.value
|
||||
if (currentPage > 1) q.p = String(currentPage)
|
||||
router.replace({ query: q }).catch(() => {})
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, computed } from 'vue'
|
||||
import { ref, computed, onBeforeUnmount } from 'vue'
|
||||
import { useRouter } from 'vue-router'
|
||||
import { NButton, NInput, NSelect, NIcon, useMessage } from 'naive-ui'
|
||||
import { SearchOutline, CodeSlashOutline, AddOutline } from '@vicons/ionicons5'
|
||||
@@ -86,8 +86,9 @@ const translated = computed(() => {
|
||||
}
|
||||
}
|
||||
// Date range: YYYY:YYYY[DP] or YYYY/MM/DD:YYYY/MM/DD[DP]
|
||||
// 对 displayText(已展开 #N)扫描,确保历史引用中的 DP 也能被翻译
|
||||
const yrRe = /(\d{4}(?:\/\d{2}\/\d{2})?)\s*:\s*(\d{4}(?:\/\d{2}\/\d{2})?)\s*\[DP\]/g
|
||||
while ((m = yrRe.exec(queryText.value)) !== null) {
|
||||
while ((m = yrRe.exec(displayText)) !== null) {
|
||||
const key = `dp_range_${m.index}`
|
||||
if (!seen.has(key)) {
|
||||
seen.add(key)
|
||||
@@ -216,20 +217,34 @@ function removeEntry(id: string) {
|
||||
remove(id)
|
||||
}
|
||||
|
||||
// ── resolve #N ──
|
||||
// ── resolve #N(迭代展开,支持嵌套引用) ──
|
||||
function resolveQuery(q: string): string {
|
||||
if (!q.includes('#')) return q
|
||||
const all = historyEntries.value
|
||||
// 只替换不在引号字符串内的 #N 引用
|
||||
return q.replace(/"[^"]*"|'[^']*'|#(\d+)/g, (m, num) => {
|
||||
if (num === undefined) return m // 在引号内,不做替换
|
||||
const found = all.find(e => e.id === `#${num}`)
|
||||
if (!found) {
|
||||
message.warning(`查询编号 ${m} 在历史中不存在,已保留原样`)
|
||||
return m
|
||||
const seen = new Set<string>()
|
||||
let prev = ''
|
||||
let current = q
|
||||
for (let i = 0; i < 10; i++) {
|
||||
if (current === prev) break
|
||||
const refs = current.match(/#\d+/g)
|
||||
if (refs) {
|
||||
for (const ref of refs) {
|
||||
if (seen.has(ref)) return prev
|
||||
seen.add(ref)
|
||||
}
|
||||
}
|
||||
return `(${found.expanded_query})`
|
||||
})
|
||||
prev = current
|
||||
current = current.replace(/"[^"]*"|'[^']*'|#(\d+)/g, (m, num) => {
|
||||
if (num === undefined) return m // 在引号内,不做替换
|
||||
const found = all.find(e => e.id === `#${num}`)
|
||||
if (!found) {
|
||||
message.warning(`查询编号 ${m} 在历史中不存在,已保留原样`)
|
||||
return m
|
||||
}
|
||||
return `(${found.expanded_query})`
|
||||
})
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
function validateQuery(q: string): { valid: boolean; query: string; error?: string } {
|
||||
@@ -287,17 +302,18 @@ function validateQuery(q: string): { valid: boolean; query: string; error?: stri
|
||||
}
|
||||
}
|
||||
|
||||
// 校验括号匹配
|
||||
// 校验括号匹配(在展开后的查询上检查,确保历史引用展开后也平衡)
|
||||
const expandedForCheck = resolveQuery(q)
|
||||
let depth = 0
|
||||
for (const ch of q) {
|
||||
for (const ch of expandedForCheck) {
|
||||
if (ch === '(') depth++
|
||||
if (ch === ')') depth--
|
||||
if (depth < 0) {
|
||||
return { valid: false, query: q, error: '括号不匹配:多余的右括号' }
|
||||
return { valid: false, query: q, error: '括号不匹配:多余的右括号(结合历史查询展开后)' }
|
||||
}
|
||||
}
|
||||
if (depth !== 0) {
|
||||
return { valid: false, query: q, error: '括号不匹配:缺少右括号' }
|
||||
return { valid: false, query: q, error: '括号不匹配:缺少右括号(结合历史查询展开后)' }
|
||||
}
|
||||
|
||||
return { valid: true, query: resolveQuery(q) }
|
||||
@@ -305,12 +321,17 @@ function validateQuery(q: string): { valid: boolean; query: string; error?: stri
|
||||
|
||||
const loading = ref(false)
|
||||
const reversedHistory = computed(() => historyEntries.value.slice().reverse())
|
||||
const fetchCountController = ref<AbortController | null>(null)
|
||||
|
||||
async function fetchCount(q: string): Promise<number | null> {
|
||||
fetchCountController.value?.abort()
|
||||
fetchCountController.value = new AbortController()
|
||||
try {
|
||||
const { data } = await api.post('/features/search/advanced', { query: q, page_size: 1 })
|
||||
const { data } = await api.post('/features/search/advanced', { query: q, page_size: 1 }, { signal: fetchCountController.value.signal })
|
||||
return (data && typeof data.total === 'number') ? data.total : null
|
||||
} catch {
|
||||
} catch (e: any) {
|
||||
if (e?.name === 'CanceledError' || e?.code === 'ERR_CANCELED') return null
|
||||
console.warn('fetchCount failed:', e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
@@ -361,6 +382,10 @@ function formatDate(ts: string): string {
|
||||
return d.toLocaleDateString('zh-CN', { month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', hour12: false })
|
||||
} catch { return ts }
|
||||
}
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
fetchCountController.value?.abort()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
@@ -423,7 +448,7 @@ function formatDate(ts: string): string {
|
||||
Search
|
||||
</NButton>
|
||||
<NButton @click="clearQuery" class="sky-btn" size="small">Clear</NButton>
|
||||
<NButton @click="addToHistory" class="sky-btn" size="small">
|
||||
<NButton @click="addToHistory" class="sky-btn" size="small" :disabled="loading">
|
||||
<template #icon><NIcon size="14"><AddOutline /></NIcon></template>
|
||||
Add to History
|
||||
</NButton>
|
||||
|
||||
@@ -38,9 +38,13 @@ const searchTotal = ref(0)
|
||||
const cursorVal = ref<string | null>(null)
|
||||
const cursorId = ref<string | null>(null)
|
||||
const hasMoreItems = ref(false)
|
||||
// offset 分页计数器(用于 best_match/relevance 排序)
|
||||
const loadMorePage = ref(1)
|
||||
const KEYSET_SORTS = new Set(['date', 'cited', 'title', 'journal', 'first_author'])
|
||||
|
||||
// 搜索请求 AbortController,防竞态
|
||||
const searchController = ref<AbortController | null>(null)
|
||||
const loadMoreController = ref<AbortController | null>(null)
|
||||
|
||||
// ── 公开统计 ──
|
||||
const platformStats = ref({ literature_total: 0, journal_total: 0, daily_avg_30d: 0 })
|
||||
@@ -144,6 +148,7 @@ async function fetchData(resetPage = true) {
|
||||
if (resetPage) {
|
||||
cursorVal.value = null
|
||||
cursorId.value = null
|
||||
loadMorePage.value = 1
|
||||
}
|
||||
loading.value = true
|
||||
try {
|
||||
@@ -319,32 +324,46 @@ function onResize() {
|
||||
isMobile.value = window.innerWidth < 768
|
||||
}
|
||||
|
||||
async function loadMore() {
|
||||
if (loadingMore.value || !hasMore.value) return
|
||||
loadingMore.value = true
|
||||
try {
|
||||
const body: Record<string, any> = { page_size: searchParams.value.page_size, sort: searchParams.value.sort || 'date' }
|
||||
if (searchParams.value.query) body.query = searchParams.value.query
|
||||
if (searchParams.value.field !== 'all') body.field = searchParams.value.field
|
||||
if (searchParams.value.tag_ids.length) body.tag_ids = searchParams.value.tag_ids
|
||||
if (searchParams.value.date_from) body.date_from = searchParams.value.date_from
|
||||
if (searchParams.value.date_to) body.date_to = searchParams.value.date_to
|
||||
if (searchParams.value.retracted) body.retracted = searchParams.value.retracted
|
||||
if (searchParams.value.negative_result) body.negative_result = searchParams.value.negative_result
|
||||
// P1-F4: 发 cursor_val(非 cursor_date),服务端兼容两者
|
||||
if (cursorVal.value && cursorId.value) {
|
||||
body.cursor_val = cursorVal.value
|
||||
body.cursor_id = cursorId.value
|
||||
}
|
||||
const { data } = await api.post('/features/search/advanced', body)
|
||||
feedItems.value.push(...(data.items || []))
|
||||
hasMoreItems.value = data.has_more ?? false
|
||||
// P1-F9: 统一使用服务端返回的游标(替代手动从末条提取)
|
||||
cursorVal.value = data.cursor_val ?? null
|
||||
cursorId.value = data.cursor_id ?? null
|
||||
} catch (e) { toast.apiError(e, '加载更多失败,请重试') }
|
||||
finally { loadingMore.value = false }
|
||||
}
|
||||
async function loadMore() {
|
||||
if (loadingMore.value || !hasMore.value) return
|
||||
loadingMore.value = true
|
||||
// 取消上次未完成的 loadMore 请求,防竞态
|
||||
loadMoreController.value?.abort()
|
||||
const controller = new AbortController()
|
||||
loadMoreController.value = controller
|
||||
try {
|
||||
const body: Record<string, any> = { page_size: searchParams.value.page_size, sort: searchParams.value.sort || 'date' }
|
||||
if (searchParams.value.query) body.query = searchParams.value.query
|
||||
if (searchParams.value.field !== 'all') body.field = searchParams.value.field
|
||||
if (searchParams.value.tag_ids.length) body.tag_ids = searchParams.value.tag_ids
|
||||
if (searchParams.value.date_from) body.date_from = searchParams.value.date_from
|
||||
if (searchParams.value.date_to) body.date_to = searchParams.value.date_to
|
||||
if (searchParams.value.retracted) body.retracted = searchParams.value.retracted
|
||||
if (searchParams.value.negative_result) body.negative_result = searchParams.value.negative_result
|
||||
const isKeyset = KEYSET_SORTS.has(searchParams.value.sort || 'date')
|
||||
if (isKeyset) {
|
||||
if (cursorVal.value && cursorId.value) {
|
||||
body.cursor_val = cursorVal.value
|
||||
body.cursor_id = cursorId.value
|
||||
}
|
||||
} else {
|
||||
body.page = loadMorePage.value + 1
|
||||
}
|
||||
const { data } = await api.post('/features/search/advanced', body, { signal: controller.signal })
|
||||
feedItems.value.push(...(data.items || []))
|
||||
hasMoreItems.value = data.has_more ?? false
|
||||
if (isKeyset) {
|
||||
cursorVal.value = data.cursor_val ?? null
|
||||
cursorId.value = data.cursor_id ?? null
|
||||
} else {
|
||||
loadMorePage.value++
|
||||
}
|
||||
} catch (e) {
|
||||
if (e instanceof DOMException && e.name === 'AbortError') return
|
||||
toast.apiError(e, '加载更多失败,请重试')
|
||||
}
|
||||
finally { loadingMore.value = false }
|
||||
}
|
||||
|
||||
// ── 从 URL 恢复状态 ──
|
||||
function restoreFromUrl() {
|
||||
@@ -418,6 +437,8 @@ onBeforeUnmount(() => {
|
||||
window.removeEventListener('scroll', onScroll)
|
||||
window.removeEventListener('resize', onResize)
|
||||
document.removeEventListener('mousedown', onDocumentMouseDown)
|
||||
searchController.value?.abort()
|
||||
loadMoreController.value?.abort()
|
||||
})
|
||||
</script>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user