Compare commits
26
Commits
80915f28a9
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2c6ecc1f91 | ||
|
|
2b7e56b9b3 | ||
|
|
41b83e3a64 | ||
|
|
3c85ded216 | ||
|
|
0197867153 | ||
|
|
fc53a5b255 | ||
|
|
475d6bb80c | ||
|
|
2b085d3445 | ||
|
|
d064dcd0ca | ||
|
|
af764237fe | ||
|
|
8dd1779676 | ||
|
|
f8db720419 | ||
|
|
c1674c602d | ||
|
|
2444a4be2c | ||
|
|
c68aa060f0 | ||
|
|
53cf1d6b70 | ||
|
|
5dcf639b0b | ||
|
|
2b6ffd6de4 | ||
|
|
1401bb7173 | ||
|
|
5fa2fbec7b | ||
|
|
de1f4a4bb8 | ||
|
|
1f46f05586 | ||
|
|
7dfde12e46 | ||
|
|
daf169d265 | ||
|
|
c5f2f27d7f | ||
|
|
1d345359fd |
@@ -46,5 +46,3 @@ backend/data/
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# Docker
|
||||
.dockerignore
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
oncolit.gonsun.com {
|
||||
reverse_proxy frontend:80
|
||||
}
|
||||
|
||||
gitea.oncolit.gonsun.com {
|
||||
reverse_proxy gitea:3000
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
.env
|
||||
.git
|
||||
.gitignore
|
||||
README.md
|
||||
test.db
|
||||
dev.db
|
||||
node_modules
|
||||
.venv
|
||||
venv
|
||||
*.log
|
||||
+11
-2
@@ -1,14 +1,23 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# 构建加速(2026-08-10):腾讯 pip/apt 源 + BuildKit pip 缓存
|
||||
# 首次 build:基镜像(daemon registry-mirrors 已配腾讯)+pip 全量(腾讯源) ≈ 几分钟
|
||||
# 之后代码级 build:pip 层命中缓存,只重算 COPY . . → 秒~1 分钟
|
||||
|
||||
# ---- Build stage ----
|
||||
FROM python:3.12-slim AS builder
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
RUN sed -i 's|deb.debian.org|mirrors.tencentyun.com|g; s|security.debian.org|mirrors.tencentyun.com|g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \
|
||||
sed -i 's|deb.debian.org|mirrors.tencentyun.com|g; s|security.debian.org|mirrors.tencentyun.com|g' /etc/apt/sources.list; \
|
||||
apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential libpq-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN printf '[global]\nindex-url = http://mirrors.tencentyun.com/pypi/simple\ntrusted-host = mirrors.tencentyun.com\n' > /etc/pip.conf
|
||||
|
||||
COPY pyproject.toml .
|
||||
RUN pip install --no-cache-dir .
|
||||
RUN --mount=type=cache,target=/root/.cache/pip pip install .
|
||||
|
||||
# ---- Runtime stage ----
|
||||
FROM python:3.12-slim
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
sqlalchemy.url = postgresql://scilit:scilit_dev@localhost:5432/scilit
|
||||
# Commit each migration in its own transaction (default: whole upgrade in one).
|
||||
# Rewrite-heavy migrations (JSON->JSONB / ALTER COLUMN TYPE DATE) on the
|
||||
# 1.23M-row prod table peak at huge disk usage; one shared txn accumulates
|
||||
# them -> disk-full (2026-08-10 batch 1 incident root cause).
|
||||
# Independent commits release space early.
|
||||
transaction_per_migration = true
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
@@ -21,19 +21,19 @@ DATE_FIELDS = ["pubmed_revised", "date_completed", "meshed_date", "create_date",
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for col in DATE_FIELDS:
|
||||
op.alter_column("global_literature", col,
|
||||
existing_type=postgresql.TIMESTAMP(timezone=True),
|
||||
type_=sa.Date(),
|
||||
existing_nullable=True,
|
||||
postgresql_using=f"{col}::date",
|
||||
)
|
||||
# 合并为单条 ALTER:5 列一次全表重写。
|
||||
# 原实现循环逐个 ALTER = 5 次整表重写,同一事务内空间峰值叠加,
|
||||
# 生产 123 万行表(~9.8GB)会爆盘(2026-08-10 批次 1 事故根因)。
|
||||
_cols = ", ".join(
|
||||
f'ALTER COLUMN "{col}" TYPE DATE USING "{col}"::date'
|
||||
for col in DATE_FIELDS
|
||||
)
|
||||
op.execute(f"ALTER TABLE global_literature {_cols}")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for col in reverse(DATE_FIELDS):
|
||||
op.alter_column("global_literature", col,
|
||||
existing_type=sa.Date(),
|
||||
type_=postgresql.TIMESTAMP(timezone=True),
|
||||
existing_nullable=True,
|
||||
)
|
||||
_cols = ", ".join(
|
||||
f'ALTER COLUMN "{col}" TYPE TIMESTAMP WITH TIME ZONE'
|
||||
for col in reversed(DATE_FIELDS)
|
||||
)
|
||||
op.execute(f"ALTER TABLE global_literature {_cols}")
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""change journal_iso/pages from varchar(100) to text
|
||||
|
||||
Revision ID: 55105f0bb1d7
|
||||
Revises: 95c18ebf31e4
|
||||
Create Date: 2026-07-30 08:38:08.376834
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = '55105f0bb1d7'
|
||||
down_revision: Union[str, None] = '95c18ebf31e4'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column('global_literature', 'journal_iso',
|
||||
existing_type=sa.VARCHAR(length=100),
|
||||
type_=sa.Text(),
|
||||
existing_nullable=True)
|
||||
op.alter_column('global_literature', 'pages',
|
||||
existing_type=sa.VARCHAR(length=100),
|
||||
type_=sa.Text(),
|
||||
existing_nullable=True)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column('global_literature', 'pages',
|
||||
existing_type=sa.Text(),
|
||||
type_=sa.VARCHAR(length=100),
|
||||
existing_nullable=True)
|
||||
op.alter_column('global_literature', 'journal_iso',
|
||||
existing_type=sa.Text(),
|
||||
type_=sa.VARCHAR(length=100),
|
||||
existing_nullable=True)
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,41 @@
|
||||
"""change volume/issue from varchar(200) to text
|
||||
|
||||
Revision ID: 95c18ebf31e4
|
||||
Revises: d166cde6083b
|
||||
Create Date: 2026-07-30 08:02:07.251190
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = '95c18ebf31e4'
|
||||
down_revision: Union[str, None] = 'd166cde6083b'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column('global_literature', 'volume',
|
||||
existing_type=sa.VARCHAR(length=200),
|
||||
type_=sa.Text(),
|
||||
existing_nullable=True)
|
||||
op.alter_column('global_literature', 'issue',
|
||||
existing_type=sa.VARCHAR(length=200),
|
||||
type_=sa.Text(),
|
||||
existing_nullable=True)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column('global_literature', 'issue',
|
||||
existing_type=sa.Text(),
|
||||
type_=sa.VARCHAR(length=200),
|
||||
existing_nullable=True)
|
||||
op.alter_column('global_literature', 'volume',
|
||||
existing_type=sa.Text(),
|
||||
type_=sa.VARCHAR(length=200),
|
||||
existing_nullable=True)
|
||||
# ### end Alembic commands ###
|
||||
@@ -5,8 +5,7 @@
|
||||
2. 新增 3 列(print_date [PPDAT], create_date [CRDT], entrez_date [EDAT])
|
||||
3. 新增 entry_terms 到 global_tags
|
||||
4. search_tsv 触发器更新(setweight A/B + authors 已为 jsonb 不再需要 CAST)
|
||||
5. search_tsv 回填全部已有记录
|
||||
6. 新增搜索索引(B-tree × 4 + GIN × 4)
|
||||
5. 新增搜索索引(B-tree × 4 + GIN × 4)
|
||||
|
||||
Revision ID: cb07d6b1df01
|
||||
Revises: 6b662a8c5235
|
||||
@@ -36,17 +35,6 @@ _TSVEC = """setweight(to_tsvector('english', COALESCE(NEW.title, '')), 'A') ||
|
||||
'')
|
||||
), 'A')"""
|
||||
|
||||
_UPDATE = """setweight(to_tsvector('english', COALESCE(title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', COALESCE(abstract, '')), 'B') ||
|
||||
setweight(to_tsvector('english',
|
||||
COALESCE(
|
||||
(SELECT string_agg(
|
||||
value->>'family' || ' ' || COALESCE(value->>'affiliation', ''),
|
||||
' ')
|
||||
FROM jsonb_array_elements(authors)),
|
||||
'')
|
||||
), 'A')"""
|
||||
|
||||
# 需改为 JSONB 的列列表
|
||||
_JSON_TO_JSONB_COLS = [
|
||||
"authors", "pub_types", "mesh_headings", "keywords", "grants",
|
||||
@@ -96,12 +84,8 @@ def upgrade() -> None:
|
||||
EXECUTE FUNCTION update_literature_search_tsv()
|
||||
""")
|
||||
|
||||
# ─── 5. 回填全部已有记录的 search_tsv ───
|
||||
op.execute(f"""
|
||||
UPDATE global_literature
|
||||
SET search_tsv = {_UPDATE}
|
||||
WHERE search_tsv IS NOT NULL;
|
||||
""")
|
||||
# ─── 5. 跳过 search_tsv 回填。后续迁移 g0h1i2j3k4l5 会做全量回填,
|
||||
# 此处回填会被完全覆盖,属于无效 I/O。
|
||||
|
||||
# ─── 6. 新增 B-tree 索引(先删后建,应对部分已存在的索引)───
|
||||
op.execute("DROP INDEX IF EXISTS ix_gl_doi")
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""extend volume/issue to varchar(200)
|
||||
|
||||
Revision ID: d166cde6083b
|
||||
Revises: af4a8b2ec873
|
||||
Create Date: 2026-07-30 00:57:23.198425
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
revision: str = 'd166cde6083b'
|
||||
down_revision: Union[str, None] = 'af4a8b2ec873'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column('global_literature', 'volume',
|
||||
existing_type=sa.VARCHAR(length=50),
|
||||
type_=sa.String(length=200),
|
||||
existing_nullable=True)
|
||||
op.alter_column('global_literature', 'issue',
|
||||
existing_type=sa.VARCHAR(length=50),
|
||||
type_=sa.String(length=200),
|
||||
existing_nullable=True)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column('global_literature', 'issue',
|
||||
existing_type=sa.String(length=200),
|
||||
type_=sa.VARCHAR(length=50),
|
||||
existing_nullable=True)
|
||||
op.alter_column('global_literature', 'volume',
|
||||
existing_type=sa.String(length=200),
|
||||
type_=sa.VARCHAR(length=50),
|
||||
existing_nullable=True)
|
||||
# ### end Alembic commands ###
|
||||
@@ -17,9 +17,9 @@ depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(op.f('ix_audit_action'), table_name='audit_logs')
|
||||
op.drop_index(op.f('ix_audit_actor'), table_name='audit_logs')
|
||||
op.drop_table('audit_logs')
|
||||
# audit_logs 保留:该表有生产数据且 admin_roles.py 通过 audit_log() 写入。
|
||||
# 模型 (app/models/audit.py) 虽未在 __init__.py 导入,但代码中活跃使用。
|
||||
# autogenerate 误判为孤立表而生成 DROP,此处移除 DROP 操作。
|
||||
op.add_column('global_literature', sa.Column('tag_ids', postgresql.ARRAY(sa.Uuid()), nullable=True))
|
||||
op.drop_index(op.f('ix_gl_authors_text_trgm'), table_name='global_literature', postgresql_ops={'(authors::text)': 'gin_trgm_ops'}, postgresql_using='gin')
|
||||
op.create_index('ix_gl_is_negative_true', 'global_literature', ['is_negative_result'], unique=False, postgresql_where=sa.text('is_negative_result = TRUE'))
|
||||
@@ -35,6 +35,7 @@ def upgrade() -> None:
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
# audit_logs 操作已从 upgrade 移除(保留生产数据),downgrade 同步跳过
|
||||
op.drop_index('ix_gl_tag_ids_gin', table_name='global_literature', postgresql_using='gin')
|
||||
op.drop_index('ix_gl_retracted_true', table_name='global_literature', postgresql_where=sa.text('retracted = TRUE'))
|
||||
op.drop_index('ix_gl_pub_year_brin', table_name='global_literature', postgresql_using='brin')
|
||||
@@ -45,18 +46,4 @@ def downgrade() -> None:
|
||||
op.drop_index('ix_gl_is_negative_true', table_name='global_literature', postgresql_where=sa.text('is_negative_result = TRUE'))
|
||||
op.create_index(op.f('ix_gl_authors_text_trgm'), 'global_literature', [sa.literal_column('(authors::text)')], unique=False, postgresql_ops={'(authors::text)': 'gin_trgm_ops'}, postgresql_using='gin')
|
||||
op.drop_column('global_literature', 'tag_ids')
|
||||
op.create_table('audit_logs',
|
||||
sa.Column('id', sa.UUID(), autoincrement=False, nullable=False),
|
||||
sa.Column('actor_id', sa.UUID(), autoincrement=False, nullable=False),
|
||||
sa.Column('tenant_id', sa.UUID(), autoincrement=False, nullable=True),
|
||||
sa.Column('action', sa.VARCHAR(length=50), autoincrement=False, nullable=False),
|
||||
sa.Column('target_type', sa.VARCHAR(length=50), autoincrement=False, nullable=True),
|
||||
sa.Column('target_id', sa.VARCHAR(length=100), autoincrement=False, nullable=True),
|
||||
sa.Column('detail', sa.TEXT(), autoincrement=False, nullable=True),
|
||||
sa.Column('ip_address', sa.VARCHAR(length=45), autoincrement=False, nullable=True),
|
||||
sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('audit_logs_pkey'))
|
||||
)
|
||||
op.create_index(op.f('ix_audit_actor'), 'audit_logs', ['actor_id', 'created_at'], unique=False)
|
||||
op.create_index(op.f('ix_audit_action'), 'audit_logs', ['action', 'created_at'], unique=False)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
+23
-15
@@ -174,27 +174,35 @@ async def dashboard_stats():
|
||||
async with async_session() as db:
|
||||
tc = (await db.execute(select(func.count(Tenant.id)))).scalar() or 0
|
||||
uc = (await db.execute(select(func.count(User.id)))).scalar() or 0
|
||||
lc = (await db.execute(select(func.count(GlobalLiterature.id)))).scalar() or 0
|
||||
nc = (await db.execute(select(func.count(UserNote.id)))).scalar() or 0
|
||||
sc = (await db.execute(select(func.count(UserLiterature.id)))).scalar() or 0
|
||||
wau = (await db.execute(select(func.count(func.distinct(LoginLog.user_id))).where(LoginLog.created_at >= week_ago))).scalar() or 0
|
||||
recent_pipelines = (await db.execute(select(PipelineRun).order_by(PipelineRun.created_at.desc()).limit(5))).scalars().all()
|
||||
r = await db.execute(select(func.count(GlobalLiterature.id)).where(GlobalLiterature.abstract.isnot(None), GlobalLiterature.abstract != ""))
|
||||
abstract_ok = r.scalar() or 0
|
||||
r = await db.execute(select(func.count(GlobalLiterature.id)).where(
|
||||
(GlobalLiterature.full_text_sections.isnot(None)) | (GlobalLiterature.full_text_path.isnot(None))
|
||||
))
|
||||
fulltext_ok = r.scalar() or 0
|
||||
# 合并所有 global_literature 聚合到一个查询(1次全扫描 vs 7次独立扫描)
|
||||
lit_agg = (await db.execute(
|
||||
select(
|
||||
func.count(GlobalLiterature.id).label("lc"),
|
||||
func.count(GlobalLiterature.id).filter(
|
||||
GlobalLiterature.abstract.isnot(None), GlobalLiterature.abstract != ""
|
||||
).label("abstract_ok"),
|
||||
func.count(GlobalLiterature.id).filter(
|
||||
(GlobalLiterature.full_text_sections.isnot(None)) | (GlobalLiterature.full_text_path.isnot(None))
|
||||
).label("fulltext_ok"),
|
||||
func.count(GlobalLiterature.id).filter(GlobalLiterature.pico.isnot(None)).label("pico_ok"),
|
||||
func.count(GlobalLiterature.id).filter(GlobalLiterature.study_design.isnot(None)).label("design_ok"),
|
||||
func.count(GlobalLiterature.id).filter(GlobalLiterature.cited_by_count.isnot(None)).label("cited_ok"),
|
||||
func.count(GlobalLiterature.id).filter(GlobalLiterature.doi.isnot(None)).label("doi_ok"),
|
||||
)
|
||||
)).one()
|
||||
lc = lit_agg.lc or 0
|
||||
abstract_ok = lit_agg.abstract_ok or 0
|
||||
fulltext_ok = lit_agg.fulltext_ok or 0
|
||||
pico_ok = lit_agg.pico_ok or 0
|
||||
design_ok = lit_agg.design_ok or 0
|
||||
cited_ok = lit_agg.cited_ok or 0
|
||||
doi_ok = lit_agg.doi_ok or 0
|
||||
r = await db.execute(select(func.count(func.distinct(GlobalLiteratureTag.literature_id))))
|
||||
tagged = r.scalar() or 0
|
||||
r = await db.execute(select(func.count(GlobalLiterature.id)).where(GlobalLiterature.pico.isnot(None)))
|
||||
pico_ok = r.scalar() or 0
|
||||
r = await db.execute(select(func.count(GlobalLiterature.id)).where(GlobalLiterature.study_design.isnot(None)))
|
||||
design_ok = r.scalar() or 0
|
||||
r = await db.execute(select(func.count(GlobalLiterature.id)).where(GlobalLiterature.cited_by_count.isnot(None)))
|
||||
cited_ok = r.scalar() or 0
|
||||
r = await db.execute(select(func.count(GlobalLiterature.id)).where(GlobalLiterature.doi.isnot(None)))
|
||||
doi_ok = r.scalar() or 0
|
||||
r = await db.execute(
|
||||
select(PipelineRun.run_type, func.count(PipelineRun.id), func.sum(PipelineRun.articles_new),
|
||||
func.sum(PipelineRun.feeds_generated))
|
||||
|
||||
@@ -279,7 +279,6 @@ async def _load_filter_options(db: AsyncSession) -> dict:
|
||||
async def advanced_search(
|
||||
req: AdvancedSearchRequest,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user: dict = Depends(get_current_user),
|
||||
):
|
||||
try:
|
||||
return await AdvancedSearchEngine.search(db, **req.model_dump())
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
"""文献 API:个性化 Feed、搜索、文献详情(需登录)"""
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi.responses import PlainTextResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import case, func, or_, select, text
|
||||
from sqlalchemy import case, func, literal, or_, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.permissions import get_current_user
|
||||
@@ -19,6 +20,8 @@ from app.services.cos_client import cached_get_full_text
|
||||
from app.services.search_engine import AdvancedSearchEngine, _escape_ilike
|
||||
from app.services.tag_loader import load_tags_for_literature
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
DISMISS_THRESHOLD = 3 # 同一标签被忽略 N 次后,Feed 引擎不再推送该标签下的文章
|
||||
|
||||
@@ -253,7 +256,6 @@ async def search_literature(
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(20, ge=1, le=100),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user: dict = Depends(get_current_user),
|
||||
):
|
||||
if not q.strip():
|
||||
return {"items": [], "total": 0}
|
||||
@@ -271,18 +273,15 @@ async def search_literature(
|
||||
return {"items": [], "total": 0, "error": "查询词过多(最多 100 个词),请简化搜索条件"}
|
||||
try:
|
||||
offset = (page - 1) * page_size
|
||||
await db.execute(text("SET LOCAL statement_timeout = '30s'"))
|
||||
like = f"%{_escape_ilike(q)}%"
|
||||
# tsvector 主搜索 + ILIKE 兜底
|
||||
search_cond = or_(
|
||||
GlobalLiterature.search_tsv.op("@@")(func.plainto_tsquery("english", q)),
|
||||
GlobalLiterature.title.ilike(like),
|
||||
GlobalLiterature.abstract.ilike(like),
|
||||
)
|
||||
await db.execute(text("SET LOCAL statement_timeout = '60s'"))
|
||||
# 主搜索:tsvector 全文检索(性能远优于多字段 OR + ILIKE)
|
||||
tsq = func.plainto_tsquery("english", q)
|
||||
search_cond = GlobalLiterature.search_tsv.op("@@")(tsq)
|
||||
# 中文搜索:自动匹配 GlobalTag.name_zh → 注入标签条件
|
||||
import re as _cn_re
|
||||
_CHINESE_RE = _cn_re.compile(r'[一-鿿㐀-䶿豈-]')
|
||||
if _CHINESE_RE.search(q):
|
||||
like = f"%{_escape_ilike(q)}%"
|
||||
_tag_matches = (await db.execute(
|
||||
select(GlobalTag.id).where(
|
||||
GlobalTag.source.in_(["mesh", "manual"]),
|
||||
@@ -291,18 +290,32 @@ async def search_literature(
|
||||
)).scalars().all()
|
||||
if _tag_matches:
|
||||
_tag_lit_subq = select(GlobalLiteratureTag.literature_id).where(
|
||||
GlobalLiteratureTag.tag_id.in_([str(t) for t in _tag_matches])
|
||||
GlobalLiteratureTag.tag_id.in_(list(_tag_matches))
|
||||
)
|
||||
search_cond = or_(search_cond, GlobalLiterature.id.in_(_tag_lit_subq))
|
||||
count_q = select(func.count(GlobalLiterature.id)).where(search_cond)
|
||||
total = (await db.execute(count_q)).scalar() or 0
|
||||
tsq = func.plainto_tsquery("english", q)
|
||||
best_match_rank = AdvancedSearchEngine._best_match_order(tsq)
|
||||
# COUNT:使用 LIMIT 10001 截断,避免大结果集的精确计数
|
||||
# 结果集 ≤10000 时返回精确值,否则返回 10001+
|
||||
MAX_EXACT = 10000
|
||||
truncated_count = await db.execute(
|
||||
select(func.count()).select_from(
|
||||
select(literal("1")).where(search_cond).limit(MAX_EXACT + 1).subquery()
|
||||
)
|
||||
)
|
||||
cnt = truncated_count.scalar() or 0
|
||||
total = cnt if cnt <= MAX_EXACT else cnt
|
||||
# ORDER BY:使用 pub_date DESC(比 best_match 公式更快且符合用户预期)
|
||||
result = await db.execute(
|
||||
select(GlobalLiterature).where(search_cond)
|
||||
.order_by(best_match_rank).offset(offset).limit(page_size)
|
||||
.order_by(GlobalLiterature.pub_date.desc().nulls_last(), GlobalLiterature.id.desc())
|
||||
.offset(offset).limit(page_size + 1)
|
||||
)
|
||||
lit_list = result.scalars().all()
|
||||
has_more = len(lit_list) > page_size
|
||||
lit_list = lit_list[:page_size]
|
||||
if page > 1 and total == 0:
|
||||
total = offset + len(lit_list) + (1 if has_more else 0)
|
||||
elif total == 0:
|
||||
total = len(lit_list)
|
||||
tm = await load_tags_for_literature(db, [str(lit.id) for lit in lit_list])
|
||||
items = []
|
||||
for lit in lit_list:
|
||||
|
||||
@@ -21,10 +21,10 @@ class GlobalLiterature(Base):
|
||||
doi: Mapped[str | None] = mapped_column(String(500))
|
||||
journal: Mapped[str | None] = mapped_column(String(500))
|
||||
journal_issn: Mapped[str | None] = mapped_column(String(20))
|
||||
journal_iso: Mapped[str | None] = mapped_column(String(100)) # NLM ISO 缩写 (e.g. "N Engl J Med")
|
||||
volume: Mapped[str | None] = mapped_column(String(50))
|
||||
issue: Mapped[str | None] = mapped_column(String(50))
|
||||
pages: Mapped[str | None] = mapped_column(String(100))
|
||||
journal_iso: Mapped[str | None] = mapped_column(Text) # NLM ISO 缩写 (e.g. "N Engl J Med")
|
||||
volume: Mapped[str | None] = mapped_column(Text)
|
||||
issue: Mapped[str | None] = mapped_column(Text)
|
||||
pages: Mapped[str | None] = mapped_column(Text)
|
||||
pub_date: Mapped[date | None] = mapped_column(Date) # 纸质出版日期(期刊卷期日期,纯电子刊则为电子日期)【PubMed: PubDate】
|
||||
print_date: Mapped[date | None] = mapped_column(Date) # 纸质出版日期(仅纸质版见刊日期)【PubMed: PPDAT】
|
||||
pub_year: Mapped[int | None] = mapped_column(Integer)
|
||||
|
||||
@@ -18,7 +18,7 @@ SMTP_PORT: int = settings.SMTP_PORT or 587
|
||||
SMTP_USER: str = settings.SMTP_USER or ""
|
||||
SMTP_PASSWORD: str = settings.SMTP_PASSWORD or ""
|
||||
FROM_EMAIL: str = settings.SMTP_FROM or "noreply@scilit-oncology.com"
|
||||
FROM_NAME: str = "OncoLit 肿瘤科研文献中心"
|
||||
FROM_NAME: str = "客户服务"
|
||||
|
||||
|
||||
async def send_email(to_email: str, subject: str, html_body: str) -> bool:
|
||||
|
||||
@@ -28,6 +28,29 @@ _PARTIAL_DATE_RE = re.compile(r'^\d{4}-\d{2}$')
|
||||
_LAST_DAY = {1:31, 2:29, 3:31, 4:30, 5:31, 6:30, 7:31, 8:31, 9:30, 10:31, 11:30, 12:31}
|
||||
|
||||
|
||||
def _is_leap_year(year: int) -> bool:
|
||||
"""Check if year is a leap year."""
|
||||
return (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0)
|
||||
|
||||
|
||||
def _validate_date_str(s: str) -> bool:
|
||||
"""Validate that a string is a valid calendar date: YYYY or YYYY-MM-DD."""
|
||||
if s.isdigit() and len(s) == 4:
|
||||
return True
|
||||
if len(s) == 10 and s[4] == '-' and s[7] == '-' and s.replace('-', '').isdigit():
|
||||
try:
|
||||
y, m, d = int(s[:4]), int(s[5:7]), int(s[8:10])
|
||||
if m < 1 or m > 12 or d < 1:
|
||||
return False
|
||||
last = _LAST_DAY.get(m, 31)
|
||||
if m == 2 and last == 29 and not _is_leap_year(y):
|
||||
last = 28
|
||||
return d <= last
|
||||
except (ValueError, IndexError):
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def _expand_partial_date(text: str) -> tuple[str, str]:
|
||||
"""Expand YYYY-MM to full month range (YYYY-MM-01 to YYYY-MM-last_day)."""
|
||||
date_from = f"{text}-01"
|
||||
@@ -37,15 +60,21 @@ def _expand_partial_date(text: str) -> tuple[str, str]:
|
||||
# P12: invalid month → return year-only range
|
||||
return f"{y}-01-01", f"{y}-12-31"
|
||||
last_day = _LAST_DAY.get(month, 31)
|
||||
if month == 2 and last_day == 29 and not _is_leap_year(int(y)):
|
||||
last_day = 28
|
||||
date_to = f"{y}-{m}-{last_day}"
|
||||
return date_from, date_to
|
||||
|
||||
|
||||
def _normalize_field_label(raw: str) -> str | None:
|
||||
"""Normalize raw PubMed field label to internal field name. (P12)"""
|
||||
if raw == "MH:NOEXP":
|
||||
if raw in ("MH:NOEXP", "MESH:NOEXP"):
|
||||
return "MH"
|
||||
return _FIELD_TAG_MAP.get(raw, _SPECIAL_FIELDS.get(raw))
|
||||
if raw in _FIELD_TAG_MAP:
|
||||
return _FIELD_TAG_MAP[raw]
|
||||
if raw in _SPECIAL_FIELDS:
|
||||
return raw
|
||||
return None
|
||||
|
||||
|
||||
# ─── 查询复杂度限制 ───
|
||||
@@ -83,6 +112,45 @@ _FIELD_TAG_MAP: dict[str, str] = {
|
||||
"FI": "GR", # [FI] Funder Identifier → 同 GR(grant_id)
|
||||
"SO": "journal", # [SO] Source → journal(近似)
|
||||
"PL": "journal", # [PL] Place of Publication → journal(近似)
|
||||
# P20: PubMed 长格式字段标签
|
||||
"TITLE": "title",
|
||||
"ABSTRACT": "abstract",
|
||||
"ALL FIELDS": "all",
|
||||
"MESH TERMS": "MH",
|
||||
"MESH MAJOR TOPIC": "MAJR",
|
||||
"TEXT WORD": "all",
|
||||
"LANGUAGE": "language",
|
||||
"AUTHOR": "author",
|
||||
"JOURNAL": "journal",
|
||||
"AFFILIATION": "affiliation",
|
||||
"PUBLICATION DATE": "DP",
|
||||
"SUBSTANCE NAME": "NM",
|
||||
"GRANT NUMBER": "GR",
|
||||
"PHARMACOLOGICAL ACTION": "PA",
|
||||
"MESH SUBHEADING": "SH",
|
||||
"PUBLICATION TYPE": "PT",
|
||||
"DATE COMPLETED": "DCOM",
|
||||
"DATE CREATED": "CRDT",
|
||||
"DATE MESH CREATED": "MHDA",
|
||||
"ENTRY DATE": "EDAT",
|
||||
"LAST REVISED": "LR",
|
||||
"DATE REVISED": "LR",
|
||||
"DATE OF ELECTRONIC PUBLICATION": "DEP",
|
||||
"SECONDARY SOURCE ID": "SI",
|
||||
"SUBSET": "SB",
|
||||
"STATUS": "STAT",
|
||||
"TRANSLITERATED TITLE": "TT",
|
||||
"VERNACULAR TITLE": "TT",
|
||||
"OTHER TERM": "OT",
|
||||
"GENE SYMBOL": "GEN",
|
||||
"PMC ID": "PMC",
|
||||
"VOLUME": "volume",
|
||||
"ISSUE": "issue",
|
||||
"PAGINATION": "pages",
|
||||
"PERSONAL NAME AS SUBJECT": "PS",
|
||||
"INVESTIGATOR": "IR",
|
||||
"CONFLICT OF INTEREST STATEMENT": "COIS",
|
||||
"AUTHOR IDENTIFIER": "AUID",
|
||||
}
|
||||
|
||||
# 需要特殊处理的字段(不直接映射到 field 参数)
|
||||
@@ -114,7 +182,7 @@ _ALL_FIELD_TAGS = {
|
||||
"DCOM", "CRDT", "EDAT", "MHDA", "LR", "DP", "DOI",
|
||||
"DEP", # P1-2: Date of Electronic Publication
|
||||
"RN", "ED", "GR", "IR", "IP",
|
||||
"TA", "JT", "LA", "LID", "MAJR", "SH", "MH", "MH:NOEXP", "OT", "PG",
|
||||
"TA", "JT", "LA", "LID", "MAJR", "SH", "MH", "MH:NOEXP", "MESH:NOEXP", "OT", "PG",
|
||||
"PA", "PT", "PMID", "PUBN", "SI", "PS", "NM", "TW",
|
||||
"SB", "STAT", "UID", # P1-2: Subset, Status, UID
|
||||
"MESH", # P1-2: [MH] 别名
|
||||
@@ -123,6 +191,21 @@ _ALL_FIELD_TAGS = {
|
||||
"Title/Abstract", # [Title/Abstract] 长标签
|
||||
"OAB", "WORD", # [OAB] Other Abstract, [WORD] Word in text
|
||||
"FI", "GEN", "PMC", "SO", "PL", # [FI] Funder, [GEN] Gene, [PMC] PMCID, [SO] Source, [PL] Place
|
||||
# P20: PubMed 长格式字段标签
|
||||
"Title", "Abstract", "All Fields",
|
||||
"MeSH Terms", "MeSH Major Topic",
|
||||
"Text Word", "Language", "Author", "Journal", "Affiliation",
|
||||
"Publication Date", "Substance Name", "Grant Number",
|
||||
"Pharmacological Action", "MeSH Subheading", "Publication Type",
|
||||
"Date Completed", "Date Created", "Date MeSH Created", "Entry Date",
|
||||
"Last Revised", "Date Revised",
|
||||
"Date of Electronic Publication",
|
||||
"Secondary Source ID", "Subset", "Status",
|
||||
"Transliterated Title", "Vernacular Title",
|
||||
"Other Term", "Gene Symbol", "PMC ID",
|
||||
"Volume", "Issue", "Pagination",
|
||||
"Personal Name as Subject", "Investigator",
|
||||
"Conflict of Interest Statement", "Author Identifier",
|
||||
}
|
||||
|
||||
|
||||
@@ -162,7 +245,7 @@ _TOKEN_PATTERNS: list[tuple[TokenType, str]] = [
|
||||
(TokenType.LPAREN, r'\('),
|
||||
(TokenType.RPAREN, r'\)'),
|
||||
(TokenType.COLON, r':'),
|
||||
(TokenType.DATE, r'\d{4}-\d{2}-\d{2}'),
|
||||
(TokenType.DATE, r'\d{4}-\d{2}(?:-\d{2})?'),
|
||||
(TokenType.NUMBER, r'\d+'),
|
||||
(TokenType.WORD, r'[^\s"\[\]():]+'),
|
||||
]
|
||||
@@ -190,15 +273,22 @@ def tokenise(query: str) -> list[Token]:
|
||||
if value is not None:
|
||||
ttype = TokenType[name]
|
||||
if ttype == TokenType.UNKNOWN_FIELD:
|
||||
fname = value.strip('[]').upper()
|
||||
# P0-1: 不认识字段标签时降级为 WORD,不终止解析
|
||||
tokens.append(Token(TokenType.WORD, value.strip('[]')))
|
||||
stripped = value.strip('[]')
|
||||
# P20: skip empty brackets like `[]`
|
||||
if not stripped:
|
||||
continue
|
||||
tokens.append(Token(TokenType.WORD, stripped))
|
||||
if len(tokens) > MAX_TERMS:
|
||||
raise ParseError(f"查询词过多(超过 {MAX_TERMS} 个),降级为简单文本搜索")
|
||||
continue
|
||||
tokens.append(Token(ttype, value))
|
||||
if len(tokens) > MAX_TERMS:
|
||||
raise ParseError(f"查询词过多(超过 {MAX_TERMS} 个),降级为简单文本搜索")
|
||||
# R23-1: capture trailing characters after last token
|
||||
if last_end < len(query):
|
||||
gap = query[last_end:]
|
||||
if gap.strip():
|
||||
tokens.append(Token(TokenType.WORD, gap.strip()))
|
||||
tokens.append(Token(TokenType.EOF))
|
||||
return tokens
|
||||
|
||||
@@ -280,8 +370,15 @@ class ParsedPubmedQuery:
|
||||
not_terms: list[Term] = field(default_factory=list) # terms under NOT
|
||||
groups: list[list[Term]] = field(default_factory=list) # parenthesized sub-groups
|
||||
group_operators: list[str] = field(default_factory=list) # "and"/"or" per group (P2-2)
|
||||
group_negated: list[bool] = field(default_factory=list) # P16: True if group was wrapped by NOT (external negation)
|
||||
sub_group_refs: list[list[int]] = field(default_factory=list) # P20: parent_gid → [child_gid, ...] for AND sub-groups
|
||||
negated_date_ranges: set[str] = field(default_factory=set) # date fields negated by NOT
|
||||
_date_range_markers: list[Term] = field(default_factory=list, repr=False) # internal: date range Term collectors
|
||||
_top_level_date_fields: set[str] = field(default_factory=set, repr=False) # date fields with ungrouped terms
|
||||
# R26: separate negated date bounds (NOT year[DP], NOT yyyy:mm[DP]) that should
|
||||
# produce independent NOT conditions instead of contaminating the positive range.
|
||||
# Keyed by field tag ("DP", "EDAT", etc.), value is list of (from_str, to_str) tuples.
|
||||
_neg_single_dates: dict[str, list[tuple[str | None, str | None]]] = field(default_factory=dict)
|
||||
|
||||
|
||||
# ─── Parser ───
|
||||
@@ -356,24 +453,37 @@ class PubmedQueryParser:
|
||||
# 同时 has_not/not_terms 也只考虑非分组词
|
||||
_ungrouped = [t for t in terms if not getattr(t, '_is_range_end', False) and t.group_id < 0]
|
||||
# P12: has_not 同时检查分组内 NOT(如 NOT (a OR b))
|
||||
# R23-3: 也检查 group_negated(NOT 包组时 is_not 被还原,group_negated 真正标记)
|
||||
result.has_not = any(t.is_not for t in _ungrouped) or any(
|
||||
t.is_not for g in result.groups for t in g
|
||||
)
|
||||
) or any(result.group_negated) or any(t.is_not for t in result._date_range_markers)
|
||||
result.not_terms = [t for t in _ungrouped if t.is_not]
|
||||
for t in _ungrouped:
|
||||
self._dispatch_term(result, t)
|
||||
# P25: track date fields with top-level (ungrouped) terms for De Morgan handling
|
||||
if t.field in _DATE_RANGE_FIELDS:
|
||||
result._top_level_date_fields.add(t.field)
|
||||
|
||||
# R27: also track top-level date range markers (excluded from _ungrouped by _is_range_end filter)
|
||||
for t in terms:
|
||||
if getattr(t, '_is_range_end', False) and t.group_id < 0:
|
||||
_tag = t.field.replace("__RANGE_", "").replace("__", "")
|
||||
if _tag in _DATE_RANGE_FIELDS:
|
||||
result._top_level_date_fields.add(_tag)
|
||||
|
||||
# P2-1: Handle unconsumed tokens (e.g., orphan text after RPAREN)
|
||||
if self.pos < len(self.tokens) - 1:
|
||||
for t in self.tokens[self.pos:-1]: # exclude EOF token
|
||||
if t.type in (TokenType.WORD, TokenType.QUOTED, TokenType.NUMBER):
|
||||
if t.type in (TokenType.WORD, TokenType.QUOTED, TokenType.NUMBER, TokenType.DATE):
|
||||
text = t.value.strip('"') if t.type == TokenType.QUOTED else t.value
|
||||
result.plain_terms.append(Term(text=text, exact=(t.type == TokenType.QUOTED)))
|
||||
|
||||
# Recompute negated_date_ranges from marker terms (after NOT toggling from recursive _parse_not_expr)
|
||||
result.negated_date_ranges = {
|
||||
# Recompute negated_date_ranges from TOP-LEVEL marker terms only.
|
||||
# R21: use |= not = to preserve negated_date_ranges added by _dispatch_term single-date NOTs
|
||||
# R28: exclude group-scoped markers (t.group_id >= 0) — engine's group loop handles them.
|
||||
result.negated_date_ranges |= {
|
||||
t.field.replace("__RANGE_", "").replace("__", "")
|
||||
for t in result._date_range_markers if t.is_not
|
||||
for t in result._date_range_markers if t.is_not and t.group_id < 0
|
||||
}
|
||||
|
||||
return result
|
||||
@@ -435,73 +545,192 @@ class PubmedQueryParser:
|
||||
# is_not 时加入 negated_date_ranges,引擎据此 NOT 条件
|
||||
elif term.field == "DP":
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
y = int(term.text)
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("DP", []).append((f"{y}-01-01", f"{y}-12-31"))
|
||||
else:
|
||||
result.year_from = max(result.year_from, y) if result.year_from is not None else y
|
||||
result.year_to = min(result.year_to, y) if result.year_to is not None else y
|
||||
elif _PARTIAL_DATE_RE.match(term.text):
|
||||
result.date_from, result.date_to = _expand_partial_date(term.text)
|
||||
df, dt = _expand_partial_date(term.text)
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("DP", []).append((df, dt))
|
||||
else:
|
||||
result.date_from = max(result.date_from, df) if result.date_from is not None else df
|
||||
result.date_to = min(result.date_to, dt) if result.date_to is not None else dt
|
||||
else:
|
||||
result.date_from = term.text
|
||||
result.date_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("DP")
|
||||
if _validate_date_str(term.text):
|
||||
df = dt = term.text
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("DP", []).append((df, dt))
|
||||
else:
|
||||
result.date_from = max(result.date_from, df) if result.date_from is not None else df
|
||||
result.date_to = min(result.date_to, dt) if result.date_to is not None else dt
|
||||
else:
|
||||
result.plain_terms.append(term)
|
||||
return
|
||||
elif term.field == "EDAT":
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
y = int(term.text)
|
||||
_f = f"{y}-01-01"
|
||||
_t = f"{y}-12-31"
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("EDAT", []).append((_f, _t))
|
||||
else:
|
||||
result.edat_from = max(result.edat_from, _f) if result.edat_from is not None else _f
|
||||
result.edat_to = min(result.edat_to, _t) if result.edat_to is not None else _t
|
||||
elif _PARTIAL_DATE_RE.match(term.text):
|
||||
result.edat_from, result.edat_to = _expand_partial_date(term.text)
|
||||
df, dt = _expand_partial_date(term.text)
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("EDAT", []).append((df, dt))
|
||||
else:
|
||||
result.edat_from = max(result.edat_from, df) if result.edat_from is not None else df
|
||||
result.edat_to = min(result.edat_to, dt) if result.edat_to is not None else dt
|
||||
else:
|
||||
result.edat_from = term.text
|
||||
result.edat_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("EDAT")
|
||||
if _validate_date_str(term.text):
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("EDAT", []).append((term.text, term.text))
|
||||
else:
|
||||
result.edat_from = max(result.edat_from, term.text) if result.edat_from is not None else term.text
|
||||
result.edat_to = min(result.edat_to, term.text) if result.edat_to is not None else term.text
|
||||
else:
|
||||
result.plain_terms.append(term)
|
||||
return
|
||||
elif term.field == "CRDT":
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
y = int(term.text)
|
||||
_f = f"{y}-01-01"
|
||||
_t = f"{y}-12-31"
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("CRDT", []).append((_f, _t))
|
||||
else:
|
||||
result.crdt_from = max(result.crdt_from, _f) if result.crdt_from is not None else _f
|
||||
result.crdt_to = min(result.crdt_to, _t) if result.crdt_to is not None else _t
|
||||
elif _PARTIAL_DATE_RE.match(term.text):
|
||||
result.crdt_from, result.crdt_to = _expand_partial_date(term.text)
|
||||
df, dt = _expand_partial_date(term.text)
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("CRDT", []).append((df, dt))
|
||||
else:
|
||||
result.crdt_from = max(result.crdt_from, df) if result.crdt_from is not None else df
|
||||
result.crdt_to = min(result.crdt_to, dt) if result.crdt_to is not None else dt
|
||||
else:
|
||||
result.crdt_from = term.text
|
||||
result.crdt_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("CRDT")
|
||||
if _validate_date_str(term.text):
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("CRDT", []).append((term.text, term.text))
|
||||
else:
|
||||
result.crdt_from = max(result.crdt_from, term.text) if result.crdt_from is not None else term.text
|
||||
result.crdt_to = min(result.crdt_to, term.text) if result.crdt_to is not None else term.text
|
||||
else:
|
||||
result.plain_terms.append(term)
|
||||
return
|
||||
elif term.field == "MHDA":
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
y = int(term.text)
|
||||
_f = f"{y}-01-01"
|
||||
_t = f"{y}-12-31"
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("MHDA", []).append((_f, _t))
|
||||
else:
|
||||
result.mhda_from = max(result.mhda_from, _f) if result.mhda_from is not None else _f
|
||||
result.mhda_to = min(result.mhda_to, _t) if result.mhda_to is not None else _t
|
||||
elif _PARTIAL_DATE_RE.match(term.text):
|
||||
df, dt = _expand_partial_date(term.text)
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("MHDA", []).append((df, dt))
|
||||
else:
|
||||
result.mhda_from = max(result.mhda_from, df) if result.mhda_from is not None else df
|
||||
result.mhda_to = min(result.mhda_to, dt) if result.mhda_to is not None else dt
|
||||
else:
|
||||
result.mhda_from = term.text
|
||||
result.mhda_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("MHDA")
|
||||
if _validate_date_str(term.text):
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("MHDA", []).append((term.text, term.text))
|
||||
else:
|
||||
result.mhda_from = max(result.mhda_from, term.text) if result.mhda_from is not None else term.text
|
||||
result.mhda_to = min(result.mhda_to, term.text) if result.mhda_to is not None else term.text
|
||||
else:
|
||||
result.plain_terms.append(term)
|
||||
return
|
||||
elif term.field == "LR":
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
y = int(term.text)
|
||||
_f = f"{y}-01-01"
|
||||
_t = f"{y}-12-31"
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("LR", []).append((_f, _t))
|
||||
else:
|
||||
result.lr_from = max(result.lr_from, _f) if result.lr_from is not None else _f
|
||||
result.lr_to = min(result.lr_to, _t) if result.lr_to is not None else _t
|
||||
elif _PARTIAL_DATE_RE.match(term.text):
|
||||
df, dt = _expand_partial_date(term.text)
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("LR", []).append((df, dt))
|
||||
else:
|
||||
result.lr_from = max(result.lr_from, df) if result.lr_from is not None else df
|
||||
result.lr_to = min(result.lr_to, dt) if result.lr_to is not None else dt
|
||||
else:
|
||||
result.lr_from = term.text
|
||||
result.lr_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("LR")
|
||||
if _validate_date_str(term.text):
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("LR", []).append((term.text, term.text))
|
||||
else:
|
||||
result.lr_from = max(result.lr_from, term.text) if result.lr_from is not None else term.text
|
||||
result.lr_to = min(result.lr_to, term.text) if result.lr_to is not None else term.text
|
||||
else:
|
||||
result.plain_terms.append(term)
|
||||
return
|
||||
elif term.field == "DCOM":
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
y = int(term.text)
|
||||
_f = f"{y}-01-01"
|
||||
_t = f"{y}-12-31"
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("DCOM", []).append((_f, _t))
|
||||
else:
|
||||
result.dcom_from = max(result.dcom_from, _f) if result.dcom_from is not None else _f
|
||||
result.dcom_to = min(result.dcom_to, _t) if result.dcom_to is not None else _t
|
||||
elif _PARTIAL_DATE_RE.match(term.text):
|
||||
df, dt = _expand_partial_date(term.text)
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("DCOM", []).append((df, dt))
|
||||
else:
|
||||
result.dcom_from = max(result.dcom_from, df) if result.dcom_from is not None else df
|
||||
result.dcom_to = min(result.dcom_to, dt) if result.dcom_to is not None else dt
|
||||
else:
|
||||
result.dcom_from = term.text
|
||||
result.dcom_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("DCOM")
|
||||
if _validate_date_str(term.text):
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("DCOM", []).append((term.text, term.text))
|
||||
else:
|
||||
result.dcom_from = max(result.dcom_from, term.text) if result.dcom_from is not None else term.text
|
||||
result.dcom_to = min(result.dcom_to, term.text) if result.dcom_to is not None else term.text
|
||||
else:
|
||||
result.plain_terms.append(term)
|
||||
return
|
||||
elif term.field == "DEP":
|
||||
if term.text.isdigit() and len(term.text) == 4:
|
||||
result.year_from = int(term.text)
|
||||
result.year_to = int(term.text)
|
||||
y = int(term.text)
|
||||
_f = f"{y}-01-01"
|
||||
_t = f"{y}-12-31"
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("DEP", []).append((_f, _t))
|
||||
else:
|
||||
result.dep_from = max(result.dep_from, _f) if result.dep_from is not None else _f
|
||||
result.dep_to = min(result.dep_to, _t) if result.dep_to is not None else _t
|
||||
elif _PARTIAL_DATE_RE.match(term.text):
|
||||
df, dt = _expand_partial_date(term.text)
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("DEP", []).append((df, dt))
|
||||
else:
|
||||
result.dep_from = max(result.dep_from, df) if result.dep_from is not None else df
|
||||
result.dep_to = min(result.dep_to, dt) if result.dep_to is not None else dt
|
||||
else:
|
||||
result.dep_from = term.text
|
||||
result.dep_to = term.text
|
||||
if term.is_not:
|
||||
result.negated_date_ranges.add("DEP")
|
||||
if _validate_date_str(term.text):
|
||||
if term.is_not:
|
||||
result._neg_single_dates.setdefault("DEP", []).append((term.text, term.text))
|
||||
else:
|
||||
result.dep_from = max(result.dep_from, term.text) if result.dep_from is not None else term.text
|
||||
result.dep_to = min(result.dep_to, term.text) if result.dep_to is not None else term.text
|
||||
else:
|
||||
result.plain_terms.append(term)
|
||||
return
|
||||
elif term.field == "__RANGE_DP__":
|
||||
pass
|
||||
elif term.field == "__RANGE_EDAT__":
|
||||
@@ -575,6 +804,23 @@ class PubmedQueryParser:
|
||||
t.group_id = gid
|
||||
result.groups.append(cluster)
|
||||
result.group_operators.append("and")
|
||||
result.group_negated.append(False) # P19: keep lengths aligned
|
||||
result.sub_group_refs.append([]) # P20: keep lengths aligned
|
||||
elif len(cluster) > 1 and any(t.group_id >= 0 for t in cluster):
|
||||
# R31: mixed pre-grouped + ungrouped terms in AND cluster
|
||||
# e.g. (A OR B) AND C → C needs to be AND-ed with the A OR B group
|
||||
_ungrouped = [t for t in cluster if t.group_id < 0]
|
||||
_child_gids = sorted(set(t.group_id for t in cluster if t.group_id >= 0))
|
||||
if _ungrouped:
|
||||
gid = len(result.groups)
|
||||
for t in _ungrouped:
|
||||
t.group_id = gid
|
||||
result.groups.append(_ungrouped)
|
||||
result.group_operators.append("and")
|
||||
result.group_negated.append(False)
|
||||
while len(result.sub_group_refs) < gid:
|
||||
result.sub_group_refs.append([])
|
||||
result.sub_group_refs.append(_child_gids if _child_gids else [])
|
||||
all_terms.extend(cluster)
|
||||
return all_terms
|
||||
|
||||
@@ -595,6 +841,8 @@ class PubmedQueryParser:
|
||||
tok = self.peek()
|
||||
if tok.type == TokenType.AND:
|
||||
self.advance()
|
||||
if self.peek().type == TokenType.EOF:
|
||||
break
|
||||
elif self._is_primary_start(tok):
|
||||
pass # implicit AND — continue without consuming
|
||||
else:
|
||||
@@ -603,18 +851,33 @@ class PubmedQueryParser:
|
||||
left.extend(right)
|
||||
return left
|
||||
|
||||
def _parse_not_expr(self, result: ParsedPubmedQuery) -> list[Term]:
|
||||
def _parse_not_expr(self, result: ParsedPubmedQuery, _not_depth: int = 0) -> list[Term]:
|
||||
"""not_expr → NOT not_expr | primary"""
|
||||
if _not_depth > MAX_PAREN_DEPTH:
|
||||
raise ParseError(f"NOT 嵌套过深(超过 {MAX_PAREN_DEPTH} 层),降级为简单文本搜索")
|
||||
if self.peek().type == TokenType.NOT:
|
||||
self.advance()
|
||||
# P5: trailing NOT at end of input → ignore silently (avoid IndexError peeking past EOF)
|
||||
if self.peek().type == TokenType.EOF:
|
||||
return []
|
||||
inner = self._parse_not_expr(result)
|
||||
inner = self._parse_not_expr(result, _not_depth + 1)
|
||||
for t in inner:
|
||||
t.is_not = not t.is_not
|
||||
# P19: NOT (A OR B) should negate the group, not individual terms.
|
||||
# When all inner terms belong to groups, revert per-term toggles
|
||||
# and set group_negated[gid] = True instead.
|
||||
if inner and all(t.group_id >= 0 for t in inner):
|
||||
for t in inner:
|
||||
t.is_not = not t.is_not # revert
|
||||
seen = set()
|
||||
for t in inner:
|
||||
gid = t.group_id
|
||||
if gid >= 0 and gid not in seen:
|
||||
seen.add(gid)
|
||||
if gid < len(result.group_negated):
|
||||
result.group_negated[gid] = not result.group_negated[gid]
|
||||
return inner
|
||||
return self._parse_primary(result, negated=False)
|
||||
return self._parse_primary(result, negated=(_not_depth % 2 == 1))
|
||||
|
||||
def _parse_primary(self, result: ParsedPubmedQuery, negated: bool = False) -> list[Term]:
|
||||
"""primary → atom FIELD? | LPAREN query RPAREN"""
|
||||
@@ -634,18 +897,54 @@ class PubmedQueryParser:
|
||||
_raw_field = ft.value[1:-1].upper()
|
||||
_field = _normalize_field_label(_raw_field)
|
||||
for t in terms:
|
||||
t.field = _field
|
||||
# 标记为子组,不放入 flat lists,保留括号分组结构
|
||||
group_id = len(result.groups)
|
||||
for t in terms:
|
||||
t.group_id = group_id
|
||||
result.groups.append(terms)
|
||||
# P2-2: 检测组内是否有显式 OR
|
||||
_has_or = any(t.type == TokenType.OR for t in self.tokens[start_pos:end_pos])
|
||||
result.group_operators.append("or" if _has_or else "and")
|
||||
if negated:
|
||||
for t in terms:
|
||||
t.is_not = True
|
||||
if not getattr(t, '_is_range_end', False): # R28: skip date range markers
|
||||
t.field = _field
|
||||
# P17: (lung OR breast)[MH:NOEXP] — 将 _noexp 传播到组内词
|
||||
if _raw_field in ("MH:NOEXP", "MESH:NOEXP"):
|
||||
for t in terms:
|
||||
t._noexp = True
|
||||
# P15-PRIMARY: 如果 _parse_or_expr 已为 AND 集群(如 A OR B AND C → [B, C] sub-group)
|
||||
# 或嵌套括号创建了子组,则这些子组已正确处理结构。
|
||||
# 只对尚未分组的词创建外层组,避免 Term 被放入两个组导致 _pubmed_conditions 重复处理。
|
||||
_parent_gid = len(result.groups)
|
||||
_ungrouped = [t for t in terms if t.group_id < 0]
|
||||
# P20: track sub-groups created by _parse_or_expr inside this paren
|
||||
# R30: exclude gids already in existing sub_group_refs (transitive children)
|
||||
# and gids that are themselves parent groups (have sub_group_refs)
|
||||
_existing_children = set(g for refs in result.sub_group_refs for g in refs)
|
||||
_child_candidates = sorted(set(t.group_id for t in terms if t.group_id >= 0))
|
||||
_child_gids = [
|
||||
gid for gid in _child_candidates
|
||||
if gid not in _existing_children
|
||||
and (gid >= len(result.sub_group_refs) or not result.sub_group_refs[gid])
|
||||
]
|
||||
for t in _ungrouped:
|
||||
t.group_id = _parent_gid
|
||||
if _ungrouped:
|
||||
# P20: ensure sub_group_refs is aligned with groups
|
||||
while len(result.sub_group_refs) < _parent_gid:
|
||||
result.sub_group_refs.append([])
|
||||
if _child_gids:
|
||||
result.sub_group_refs.append(_child_gids)
|
||||
else:
|
||||
result.sub_group_refs.append([])
|
||||
# R21: only write sub_group_refs when creating a parent group,
|
||||
# preventing phantom entries when _ungrouped is empty (all terms already in child groups)
|
||||
result.groups.append(_ungrouped)
|
||||
# R30: track paren depth to avoid counting OR inside nested parens
|
||||
_or_depth = 0
|
||||
_has_or = False
|
||||
for _t in self.tokens[start_pos:end_pos]:
|
||||
if _t.type == TokenType.LPAREN:
|
||||
_or_depth += 1
|
||||
elif _t.type == TokenType.RPAREN:
|
||||
_or_depth -= 1
|
||||
elif _t.type == TokenType.OR and _or_depth == 0:
|
||||
_has_or = True
|
||||
break
|
||||
result.group_operators.append("or" if _has_or else "and")
|
||||
result.group_negated.append(False) # P16: P19 revert logic handles NOT group tracking
|
||||
# 已分组的 Term(嵌套括号 OR 子组)不再重复加组
|
||||
return terms
|
||||
|
||||
return self._parse_atom(result, negated)
|
||||
@@ -666,23 +965,42 @@ class PubmedQueryParser:
|
||||
t1 = self.peek_n(1)
|
||||
t2 = self.peek_n(2)
|
||||
|
||||
# R29: open-ended start range :2024[DP] → COLON NUMBER/DATE FIELD
|
||||
if t0.type == TokenType.COLON and t1 is not None and t1.type in (TokenType.NUMBER, TokenType.DATE):
|
||||
self.advance() # consume COLON
|
||||
e_val = self.advance().value # end value
|
||||
return self._handle_date_range_edge(result, negated, start_val=None, end_val=e_val)
|
||||
|
||||
# R29: open-ended end range 2024:[DP] → NUMBER/DATE COLON FIELD
|
||||
if (t0.type in (TokenType.NUMBER, TokenType.DATE) and t1 is not None and t1.type == TokenType.COLON
|
||||
and t2 is not None and t2.type == TokenType.FIELD):
|
||||
s_val = self.advance().value # start value
|
||||
self.advance() # consume COLON
|
||||
return self._handle_date_range_edge(result, negated, start_val=s_val, end_val=None)
|
||||
|
||||
if (t1 is not None and t1.type == TokenType.COLON
|
||||
and t2 is not None
|
||||
and t0.type in (TokenType.NUMBER, TokenType.DATE, TokenType.WORD)
|
||||
and t2.type in (TokenType.NUMBER, TokenType.DATE, TokenType.WORD)):
|
||||
return self._parse_range(result, negated)
|
||||
|
||||
# Normal atom
|
||||
# Normal atom — only consume if peek is a valid atomic token
|
||||
t0 = self.peek()
|
||||
if t0.type not in (TokenType.WORD, TokenType.QUOTED, TokenType.NUMBER, TokenType.DATE):
|
||||
return []
|
||||
token = self.advance()
|
||||
text = token.value.strip('"') if token.type == TokenType.QUOTED else token.value
|
||||
# R21: skip empty quoted text ""[TI]; R27: also skip whitespace-only
|
||||
if not text or not text.strip():
|
||||
return []
|
||||
is_exact = (token.type == TokenType.QUOTED)
|
||||
field = None
|
||||
_noexp = False # P1-4
|
||||
if self.peek().type == TokenType.FIELD:
|
||||
ft = self.advance()
|
||||
raw = ft.value[1:-1].upper()
|
||||
# P1-4: [MH:noexp] → 抑制树展开
|
||||
if raw == "MH:NOEXP":
|
||||
# P1-4: [MH:noexp] / [MESH:noexp] → 抑制树展开
|
||||
if raw in ("MH:NOEXP", "MESH:NOEXP"):
|
||||
field = "MH"
|
||||
_noexp = True
|
||||
else:
|
||||
@@ -690,7 +1008,7 @@ class PubmedQueryParser:
|
||||
if field in _FIELD_TAG_MAP:
|
||||
field = _FIELD_TAG_MAP[field]
|
||||
|
||||
return [Term(text, exact=is_exact, field=field, is_not=negated, _noexp=_noexp)]
|
||||
return [Term(text, exact=is_exact, field=field, is_not=False, _noexp=_noexp)]
|
||||
|
||||
def _parse_range(self, result: ParsedPubmedQuery, negated: bool = False) -> list[Term]:
|
||||
"""Parse NUMBER:NUMBER[FIELD] — handles date ranges specially."""
|
||||
@@ -702,6 +1020,10 @@ class PubmedQueryParser:
|
||||
if self.peek().type == TokenType.FIELD:
|
||||
ft = self.advance()
|
||||
field = ft.value[1:-1].upper()
|
||||
# R23-2: normalize field label for non-date ranges too
|
||||
_norm = _normalize_field_label(field)
|
||||
if _norm is not None:
|
||||
field = _norm
|
||||
|
||||
if field in _DATE_RANGE_FIELDS:
|
||||
attr_map = {
|
||||
@@ -746,56 +1068,184 @@ class PubmedQueryParser:
|
||||
except IndexError:
|
||||
pass
|
||||
# P12: validate date values — non-numeric garbage falls back to plain text
|
||||
_valid_date = lambda s: (s.isdigit() and len(s) == 4) or (
|
||||
len(s) == 10 and s[4] == '-' and s[7] == '-' and s.replace('-', '').isdigit()
|
||||
)
|
||||
# R27: expand partial dates YYYY-MM in ranges before validation
|
||||
if _PARTIAL_DATE_RE.match(start_val):
|
||||
start_val, _ = _expand_partial_date(start_val)
|
||||
if _PARTIAL_DATE_RE.match(end_val):
|
||||
_, end_val = _expand_partial_date(end_val)
|
||||
_valid_date = lambda s: _validate_date_str(s)
|
||||
if not _valid_date(start_val) or not _valid_date(end_val):
|
||||
txt = f"{start_val}:{end_val}[{field}]"
|
||||
return [Term(txt, field=field, is_not=negated)]
|
||||
return [Term(txt, field=None, is_not=False)]
|
||||
# 确定两端是否是 4 位年份
|
||||
_start_is_year = start_val.isdigit() and len(start_val) == 4
|
||||
_end_is_year = end_val.isdigit() and len(end_val) == 4
|
||||
# Year-only range (e.g., 2024:2026[EDAT])
|
||||
if _start_is_year and _end_is_year:
|
||||
try:
|
||||
if yr_from_attr:
|
||||
setattr(result, yr_from_attr, int(start_val))
|
||||
setattr(result, yr_to_attr, int(end_val))
|
||||
# R27: only set global attributes for top-level ranges (depth == 0)
|
||||
# Ranges inside groups are handled via markers in the engine's group path.
|
||||
if self._depth == 0:
|
||||
if _start_is_year and _end_is_year:
|
||||
if negated:
|
||||
# R26: store in _neg_single_dates instead of main fields
|
||||
result._neg_single_dates.setdefault(field, []).append(
|
||||
(f"{start_val}-01-01", f"{end_val}-12-31")
|
||||
)
|
||||
else:
|
||||
# For non-DP date fields: convert year to full date for consistency
|
||||
setattr(result, date_attr, f"{start_val}-01-01")
|
||||
setattr(result, date_attr_to, f"{end_val}-12-31")
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
elif _start_is_year and not _end_is_year:
|
||||
# Mixed: start is year, end is full date (e.g., 2024:2024-12-01[EDAT])
|
||||
if yr_from_attr:
|
||||
try:
|
||||
setattr(result, yr_from_attr, int(start_val))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
setattr(result, date_attr, f"{start_val}-01-01")
|
||||
setattr(result, date_attr_to, end_val)
|
||||
elif not _start_is_year and _end_is_year:
|
||||
# Mixed: start is full date, end is year (e.g., 2024-01-01:2026[EDAT])
|
||||
setattr(result, date_attr, start_val)
|
||||
setattr(result, date_attr_to, f"{end_val}-12-31")
|
||||
else:
|
||||
# Full date range (e.g., 2024-01-01:2024-12-31[EDAT])
|
||||
setattr(result, date_attr, start_val)
|
||||
setattr(result, date_attr_to, end_val)
|
||||
marker = Term(f"{start_val}:{end_val}", field=marker_field, is_not=negated)
|
||||
try:
|
||||
if yr_from_attr:
|
||||
curr_f = getattr(result, yr_from_attr)
|
||||
new_f = int(start_val)
|
||||
setattr(result, yr_from_attr, max(curr_f, new_f) if curr_f is not None else new_f)
|
||||
curr_t = getattr(result, yr_to_attr)
|
||||
new_t = int(end_val)
|
||||
setattr(result, yr_to_attr, min(curr_t, new_t) if curr_t is not None else new_t)
|
||||
else:
|
||||
# For non-DP date fields: convert year to full date for consistency
|
||||
curr_f = getattr(result, date_attr)
|
||||
new_f = f"{start_val}-01-01"
|
||||
setattr(result, date_attr, max(curr_f, new_f) if curr_f is not None else new_f)
|
||||
curr_t = getattr(result, date_attr_to)
|
||||
new_t = f"{end_val}-12-31"
|
||||
setattr(result, date_attr_to, min(curr_t, new_t) if curr_t is not None else new_t)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
elif _start_is_year and not _end_is_year:
|
||||
# Mixed: start is year, end is full date (e.g., 2024:2024-12-01[EDAT])
|
||||
if negated:
|
||||
result._neg_single_dates.setdefault(field, []).append(
|
||||
(f"{start_val}-01-01", end_val)
|
||||
)
|
||||
else:
|
||||
if yr_from_attr:
|
||||
try:
|
||||
curr = getattr(result, yr_from_attr)
|
||||
v = int(start_val)
|
||||
setattr(result, yr_from_attr, max(curr, v) if curr is not None else v)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
curr_f = getattr(result, date_attr)
|
||||
new_f = f"{start_val}-01-01"
|
||||
setattr(result, date_attr, max(curr_f, new_f) if curr_f is not None else new_f)
|
||||
curr_t = getattr(result, date_attr_to)
|
||||
setattr(result, date_attr_to, min(curr_t, end_val) if curr_t is not None else end_val)
|
||||
elif not _start_is_year and _end_is_year:
|
||||
# Mixed: start is full date, end is year (e.g., 2024-01-01:2026[EDAT])
|
||||
if negated:
|
||||
result._neg_single_dates.setdefault(field, []).append(
|
||||
(start_val, f"{end_val}-12-31")
|
||||
)
|
||||
else:
|
||||
curr_f = getattr(result, date_attr)
|
||||
setattr(result, date_attr, max(curr_f, start_val) if curr_f is not None else start_val)
|
||||
curr_t = getattr(result, date_attr_to)
|
||||
v = f"{end_val}-12-31"
|
||||
setattr(result, date_attr_to, min(curr_t, v) if curr_t is not None else v)
|
||||
else:
|
||||
# Full date range (e.g., 2024-01-01:2024-12-31[EDAT])
|
||||
if negated:
|
||||
result._neg_single_dates.setdefault(field, []).append((start_val, end_val))
|
||||
else:
|
||||
curr_f = getattr(result, date_attr)
|
||||
setattr(result, date_attr, max(curr_f, start_val) if curr_f is not None else start_val)
|
||||
curr_t = getattr(result, date_attr_to)
|
||||
setattr(result, date_attr_to, min(curr_t, end_val) if curr_t is not None else end_val)
|
||||
marker = Term(f"{start_val}:{end_val}", field=marker_field, is_not=False)
|
||||
marker._is_range_end = True
|
||||
result._date_range_markers.append(marker)
|
||||
if negated:
|
||||
result.negated_date_ranges.add(field)
|
||||
return [marker]
|
||||
|
||||
# Non-date range or no field → plain text
|
||||
txt = f"{start_val}:{end_val}"
|
||||
if field:
|
||||
txt = f"{txt}[{field}]"
|
||||
return [Term(txt, field=field, is_not=negated)]
|
||||
# R29: field tag is already stored in Term.field — do not append to text
|
||||
return [Term(txt, field=field or "", is_not=False)]
|
||||
|
||||
# ─── R29: Open-ended date range helper ───
|
||||
def _handle_date_range_edge(self, result: ParsedPubmedQuery, negated: bool = False,
|
||||
start_val: str | None = None, end_val: str | None = None) -> list[Term]:
|
||||
"""Handle open-ended ranges like :2024[DP] or 2024:[DP]."""
|
||||
field = None
|
||||
if self.peek().type == TokenType.FIELD:
|
||||
ft = self.advance()
|
||||
field = ft.value[1:-1].upper()
|
||||
_norm = _normalize_field_label(field)
|
||||
if _norm is not None:
|
||||
field = _norm
|
||||
|
||||
if field not in _DATE_RANGE_FIELDS:
|
||||
txt = f"{start_val or ''}:{end_val or ''}"
|
||||
return [Term(txt, field=field or "", is_not=False)]
|
||||
|
||||
attr_map = {
|
||||
"DP": ("date_from", "date_to", "year_from", "year_to", "__RANGE_DP__"),
|
||||
"EDAT": ("edat_from", "edat_to", None, None, "__RANGE_EDAT__"),
|
||||
"CRDT": ("crdt_from", "crdt_to", None, None, "__RANGE_CRDT__"),
|
||||
"MHDA": ("mhda_from", "mhda_to", None, None, "__RANGE_MHDA__"),
|
||||
"LR": ("lr_from", "lr_to", None, None, "__RANGE_LR__"),
|
||||
"DCOM": ("dcom_from", "dcom_to", None, None, "__RANGE_DCOM__"),
|
||||
"DEP": ("dep_from", "dep_to", None, None, "__RANGE_DEP__"),
|
||||
}
|
||||
date_attr, date_attr_to, yr_from_attr, yr_to_attr, marker_field = attr_map[field]
|
||||
|
||||
if self._depth == 0:
|
||||
if start_val is not None and end_val is None:
|
||||
# Open-ended end: 2024:[DP] → from start_val onwards
|
||||
if start_val.isdigit() and len(start_val) == 4:
|
||||
if yr_from_attr is not None:
|
||||
if negated:
|
||||
result._neg_single_dates.setdefault(field, []).append(
|
||||
(f"{start_val}-01-01", None))
|
||||
else:
|
||||
curr = getattr(result, yr_from_attr)
|
||||
v = int(start_val)
|
||||
setattr(result, yr_from_attr, max(curr, v) if curr is not None else v)
|
||||
else:
|
||||
if negated:
|
||||
result._neg_single_dates.setdefault(field, []).append(
|
||||
(f"{start_val}-01-01", None))
|
||||
else:
|
||||
curr = getattr(result, date_attr)
|
||||
v = f"{start_val}-01-01"
|
||||
setattr(result, date_attr, max(curr, v) if curr is not None else v)
|
||||
else:
|
||||
if _PARTIAL_DATE_RE.match(start_val):
|
||||
start_val, _ = _expand_partial_date(start_val)
|
||||
if negated:
|
||||
result._neg_single_dates.setdefault(field, []).append((start_val, None))
|
||||
else:
|
||||
curr = getattr(result, date_attr)
|
||||
setattr(result, date_attr, max(curr, start_val) if curr is not None else start_val)
|
||||
elif end_val is not None and start_val is None:
|
||||
# Open-ended start: :2024[DP] → up to end_val
|
||||
if end_val.isdigit() and len(end_val) == 4:
|
||||
if yr_to_attr is not None:
|
||||
if negated:
|
||||
result._neg_single_dates.setdefault(field, []).append(
|
||||
(None, f"{end_val}-12-31"))
|
||||
else:
|
||||
curr = getattr(result, yr_to_attr)
|
||||
v = int(end_val)
|
||||
setattr(result, yr_to_attr, min(curr, v) if curr is not None else v)
|
||||
else:
|
||||
if negated:
|
||||
result._neg_single_dates.setdefault(field, []).append(
|
||||
(None, f"{end_val}-12-31"))
|
||||
else:
|
||||
curr = getattr(result, date_attr_to)
|
||||
v = f"{end_val}-12-31"
|
||||
setattr(result, date_attr_to, min(curr, v) if curr is not None else v)
|
||||
else:
|
||||
if _PARTIAL_DATE_RE.match(end_val):
|
||||
_, end_val = _expand_partial_date(end_val)
|
||||
if negated:
|
||||
result._neg_single_dates.setdefault(field, []).append((None, end_val))
|
||||
else:
|
||||
curr = getattr(result, date_attr_to)
|
||||
setattr(result, date_attr_to, min(curr, end_val) if curr is not None else end_val)
|
||||
|
||||
marker = Term(f"{start_val or ''}:{end_val or ''}", field=marker_field, is_not=False)
|
||||
marker._is_range_end = True
|
||||
result._date_range_markers.append(marker)
|
||||
return [marker]
|
||||
|
||||
|
||||
# ─── Public API ───
|
||||
@@ -811,9 +1261,9 @@ def is_pubmed_syntax(query: str) -> bool:
|
||||
return False
|
||||
# P5: Normalize fullwidth characters before checking
|
||||
query = unicodedata.normalize('NFKC', query)
|
||||
if re.search(r'\[(' + '|'.join(_ALL_FIELD_TAGS) + r')\]', query, re.IGNORECASE):
|
||||
if re.search(r'\[(' + '|'.join(_ALL_FIELD_TAGS) + r')\]', query, re.IGNORECASE | re.ASCII):
|
||||
return True
|
||||
if re.search(r'\b(AND|OR|NOT)\b', query):
|
||||
if re.search(r'\b(AND|OR|NOT)\b', query, re.IGNORECASE | re.ASCII):
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -829,21 +1279,28 @@ def parse_pubmed_query(query: str) -> ParsedPubmedQuery:
|
||||
try:
|
||||
# P2-3: Unicode normalization — strip zero-width chars, normalize fullwidth digits
|
||||
query = unicodedata.normalize('NFKC', query)
|
||||
import re as _re
|
||||
# 将 YYYY/MM/DD 或 YYYY/M/D 格式的日期分隔符统一为 YYYY-MM-DD,使 tokeniser 正确识别为 DATE
|
||||
# Anchored with context boundaries to avoid over-matching inside URLs/paths
|
||||
query = _re.sub(
|
||||
query = re.sub(
|
||||
r'(^|[\[\s":])(\d{4})/(\d{1,2})/(\d{1,2})(?=\s|$|[\[\]":])',
|
||||
lambda m: f'{m.group(1)}{m.group(2)}-{int(m.group(3)):02d}-{int(m.group(4)):02d}',
|
||||
query,
|
||||
)
|
||||
# P5: Normalize single-digit month/day (2024-1-1 → 2024-01-01) to match DATE token pattern
|
||||
# Anchored with context boundaries to avoid over-matching inside non-date text
|
||||
query = _re.sub(
|
||||
query = re.sub(
|
||||
r'(^|[\[\s":])(\d{4})-(\d{1,2})-(\d{1,2})(?=\s|$|[\[\]":])',
|
||||
lambda m: f'{m.group(1)}{m.group(2)}-{int(m.group(3)):02d}-{int(m.group(4)):02d}',
|
||||
query,
|
||||
)
|
||||
# P5: Normalize YYYY-MM (partial month) to YYYY-MM-01 when followed by date field tag
|
||||
# R23-3: support single-digit month (2024-1[DP] → 2024-01-01)
|
||||
# The \s*\[ lookahead prevents false match on YYYY-MM-DD sequences
|
||||
query = re.sub(
|
||||
r'(\b\d{4})-(\d{1,2})(?=\s*\[(?:DP|EDAT|DEP|CRDT|MHDA|LR|DCOM)\])',
|
||||
lambda m: f'{m.group(1)}-{int(m.group(2)):02d}',
|
||||
query,
|
||||
)
|
||||
tokens = tokenise(query)
|
||||
parser = PubmedQueryParser(tokens)
|
||||
return parser.parse()
|
||||
@@ -874,7 +1331,7 @@ def extract_pubmed_query_for_prisma(query: str) -> tuple[str, list[str]]:
|
||||
|
||||
# P5: Handle field tags with '/' (e.g. Title/Article) or special chars
|
||||
normalized = re.sub(
|
||||
r'\[([\w/:]+)\]',
|
||||
r'\[([\w/: -]+)\]',
|
||||
lambda m: f'[{m.group(1).upper()}]',
|
||||
query,
|
||||
)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# migrate_prod.sh — 生产数据迁移分批执行脚本
|
||||
#
|
||||
# 用法:
|
||||
# ./migrate_prod.sh <批次 1|2|3|4>
|
||||
#
|
||||
# 批次划分(对应 docs/17-生产数据迁移.md 第 4 节):
|
||||
# 1: [01-03] 基础 schema —— JSON→JSONB 全表重写、meshed_date 回填、5 日期字段→DATE(锁表)
|
||||
# 2: [04-11] 轻量加列/新表 —— pipeline_runs/journals/literature 增列 + user_saved_filters
|
||||
# 3: [12-16] 索引 + 触发器 —— 筛选列/mesh_headings/trgm 索引 + tsvector 触发器
|
||||
# 4: [17-24] 大数据量 —— author_names/search_tsv/tag_ids 回填 + volume/pages→Text(锁表)
|
||||
#
|
||||
# 前置条件(必须全部满足,否则脚本拦截):
|
||||
# ① /root/scilit/backend 已同步最新代码(含 alembic/versions/ 的新迁移文件)
|
||||
# ② 已构建新 backend 镜像:
|
||||
# docker compose -f docker-compose.prod.yml build backend
|
||||
# ③ postgres 容器 healthy
|
||||
# ④ 脚本放在 /root/scilit/ 目录执行(与 docker-compose.prod.yml 同目录)
|
||||
#
|
||||
# 说明:
|
||||
# - 每个批次执行到指定 checkpoint revision,验证后停止;下一批由人工决定何时跑
|
||||
# - 全部使用 `run --no-deps --rm`(防依赖链拉起 migrate 服务重复跑迁移)
|
||||
# - 批次 1/4 含锁表操作,建议低峰执行
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
# 数组:命令 + 参数("$COMPOSE" 双引号会把整串当一个命令名,导致 command not found)
|
||||
COMPOSE=(docker compose -f docker-compose.prod.yml)
|
||||
LOCAL_HEAD="55105f0bb1d7" # 本地最新迁移 revision([24])
|
||||
|
||||
# 批次 → 目标 revision
|
||||
declare -A TARGET=(
|
||||
[1]="1421ea169bb6" # [03] 5 日期字段→DATE
|
||||
[2]="cac545862583" # [11] user_saved_filters
|
||||
[3]="e0f1a2b3c4d5" # [16] chemical/gene tsvector 触发器
|
||||
[4]="head" # [24] journal_iso/pages→Text
|
||||
)
|
||||
# 批次 → 预期起点 revision(DB 当前应在此处,防止乱序)
|
||||
declare -A EXPECT_START=(
|
||||
[1]="6b662a8c5235" # 生产当前 head
|
||||
[2]="1421ea169bb6"
|
||||
[3]="cac545862583"
|
||||
[4]="e0f1a2b3c4d5"
|
||||
)
|
||||
declare -A DESC=(
|
||||
[1]="[01-03] 基础 schema:JSON→JSONB、meshed_date 回填、5 日期字段→DATE(⚠️ destructive:日期收窄,须与发新代码同窗口,不能脱离部署单独跑)"
|
||||
[2]="[04-11] 轻量加列/新表:pipeline_runs/journals/literature 增列 + user_saved_filters(纯 additive,可提前任意时段跑)"
|
||||
[3]="[12-16] 索引 + 触发器:筛选列/mesh_headings/trgm 索引 + tsvector 触发器(additive,可提前,建议低峰)"
|
||||
[4]="[17-24] 大数据量回填 + 类型锁表:author_names_text/search_tsv/tag_ids 回填 + volume/pages→Text(含 destructive 类型变更,须在新代码已部署后、深夜低峰)"
|
||||
)
|
||||
|
||||
get_db_rev() {
|
||||
# 返回 DB 当前 revision 的短 hash(12 位);alembic_version 表不存在则报错
|
||||
# -c alembic/alembic.ini 必须带:run 覆盖 command 后默认 cwd 不是 /app,不带找不到 script_location
|
||||
# 2>/dev/null 丢 docker 的 "Container ... Creating" stderr;tail -1 取 alembic 真正输出的最后一行
|
||||
# (否则容器 ID 的 12 位 hex 会被 grep 误抓成 revision,报 "not expected start")
|
||||
"${COMPOSE[@]}" run --no-deps --rm backend alembic -c alembic/alembic.ini current 2>/dev/null \
|
||||
| grep -oE '[0-9a-f]{12}' | tail -1
|
||||
}
|
||||
|
||||
check_image_fresh() {
|
||||
echo "── 检查 backend 镜像是否含最新迁移文件(本地 head: $LOCAL_HEAD)──"
|
||||
local heads
|
||||
heads="$("${COMPOSE[@]}" run --no-deps --rm backend alembic -c alembic/alembic.ini heads 2>/dev/null)" || true
|
||||
if ! echo "$heads" | grep -q "$LOCAL_HEAD"; then
|
||||
echo "✗ 镜像不含本地 head($LOCAL_HEAD)。"
|
||||
echo " 当前镜像 alembic heads:"
|
||||
echo "$heads" | sed 's/^/ /'
|
||||
echo ""
|
||||
echo " 请先在生产完成以下步骤再跑迁移:"
|
||||
echo " 1. 同步最新代码到 /root/scilit/backend"
|
||||
echo " 2. docker compose -f docker-compose.prod.yml build backend"
|
||||
echo " 用旧镜像跑迁移会报 Can't locate revision(8-09 事故根因)。"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ 镜像包含本地 head,可执行迁移"
|
||||
}
|
||||
|
||||
usage() {
|
||||
echo "用法: $0 <批次 1|2|3|4>"
|
||||
echo ""
|
||||
for n in 1 2 3 4; do
|
||||
echo " 批次 $n: ${DESC[$n]}"
|
||||
done
|
||||
}
|
||||
|
||||
if [[ $# -ne 1 || ! ${TARGET[$1]+x} ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
BATCH=$1
|
||||
REV=${TARGET[$BATCH]}
|
||||
START=${EXPECT_START[$BATCH]}
|
||||
|
||||
echo "════════════════════════════════════════════════════════════"
|
||||
echo " 批次 $BATCH/4:${DESC[$BATCH]}"
|
||||
echo " 目标 revision: $REV"
|
||||
echo "════════════════════════════════════════════════════════════"
|
||||
|
||||
# 前置 1: 镜像新鲜度
|
||||
check_image_fresh
|
||||
|
||||
# 前置 2: DB 起点校验(防乱序/重复)
|
||||
CUR="$(get_db_rev)"
|
||||
echo "── DB 当前 revision: $CUR ──"
|
||||
if [[ "$CUR" == "$REV" || ( "$BATCH" == "4" && "$CUR" == "$LOCAL_HEAD" ) ]]; then
|
||||
echo "✓ DB 已在该批次目标 revision,无需执行(幂等跳过)"
|
||||
exit 0
|
||||
fi
|
||||
if [[ "$CUR" != "$START" ]]; then
|
||||
echo "✗ DB 当前 revision($CUR)不是批次 $BATCH 的预期起点($START)。"
|
||||
echo " 请确认前一批次已执行完成,或检查批次顺序。"
|
||||
echo " 当前 migration 链:"
|
||||
"${COMPOSE[@]}" run --no-deps --rm backend alembic -c alembic/alembic.ini history 2>&1 | tail -30 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 执行迁移
|
||||
echo "── 开始执行迁移(可能耗时,请勿中断)──"
|
||||
"${COMPOSE[@]}" run --no-deps --rm backend alembic -c alembic/alembic.ini upgrade "$REV"
|
||||
|
||||
# 验证
|
||||
AFTER="$(get_db_rev)"
|
||||
echo "── 迁移后 DB revision: $AFTER ──"
|
||||
if [[ "$AFTER" != "$REV" && ! ( "$BATCH" == "4" && "$AFTER" == "$LOCAL_HEAD" ) ]]; then
|
||||
echo "✗ 迁移后 revision 未达到目标,异常!"
|
||||
exit 1
|
||||
fi
|
||||
echo ""
|
||||
echo "✓ 批次 $BATCH 完成。"
|
||||
if [[ $BATCH -lt 4 ]]; then
|
||||
echo " 下一步: 验证无误后,可运行 ./migrate_prod.sh $((BATCH+1))"
|
||||
else
|
||||
echo " 全部 4 个批次执行完毕,DB 已升级到 head($LOCAL_HEAD)。"
|
||||
fi
|
||||
@@ -6,12 +6,12 @@ import os
|
||||
import sys
|
||||
from datetime import date, datetime
|
||||
|
||||
from app.compat import UTC
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
||||
|
||||
import uuid
|
||||
|
||||
from app.compat import UTC
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from app.core.security import hash_password
|
||||
|
||||
@@ -345,20 +345,24 @@ class TestNegatedTerms:
|
||||
assert r.mesh_terms[0].field == "MH"
|
||||
|
||||
def test_not_group_and(self):
|
||||
"""NOT (cancer AND tumor) — group NOT should set all terms is_not"""
|
||||
"""NOT (cancer AND tumor) — group NOT should set group_negated"""
|
||||
r = parse_pubmed_query("NOT (cancer AND tumor)")
|
||||
assert len(r.groups) >= 1
|
||||
assert len(r.group_negated) >= 1
|
||||
assert r.group_negated[0] is True
|
||||
for group in r.groups:
|
||||
for term in group:
|
||||
assert term.is_not is True, f"All terms in negated group should be NOT: {group}"
|
||||
assert term.is_not is False, f"NOT group terms should have is_not=False (group_negated tracks negation): {group}"
|
||||
|
||||
def test_not_group_or(self):
|
||||
"""NOT (cancer OR tumor) — group NOT with OR"""
|
||||
r = parse_pubmed_query("NOT (cancer OR tumor)")
|
||||
assert len(r.groups) >= 1
|
||||
assert len(r.group_negated) >= 1
|
||||
assert r.group_negated[0] is True
|
||||
for group in r.groups:
|
||||
for term in group:
|
||||
assert term.is_not is True
|
||||
assert term.is_not is False
|
||||
|
||||
def test_triple_not(self):
|
||||
"""NOT NOT NOT cancer = NOT cancer"""
|
||||
|
||||
@@ -32,8 +32,8 @@ class TestIsPubmedSyntax:
|
||||
assert not is_pubmed_syntax(None)
|
||||
|
||||
def test_lowercase_boolean_detected(self):
|
||||
"""小写的 and/or/not 不识别为 PubMed 语法(P11: 仅大写 AND/OR/NOT 才是 PubMed 布尔符)"""
|
||||
assert not is_pubmed_syntax("cancer and therapy")
|
||||
"""小写的 and/or/not 识别为 PubMed 语法(P20: 添加 re.IGNORECASE,统一处理大小写)"""
|
||||
assert is_pubmed_syntax("cancer and therapy")
|
||||
|
||||
|
||||
class TestTokenise:
|
||||
|
||||
+29
-8
@@ -246,7 +246,7 @@ services:
|
||||
environment:
|
||||
VITE_API_BASE: /api/v1
|
||||
ports:
|
||||
- "80:80"
|
||||
- "127.0.0.1:8080:80"
|
||||
volumes:
|
||||
- /var/log/scilit/nginx:/var/log/nginx
|
||||
depends_on:
|
||||
@@ -260,7 +260,7 @@ services:
|
||||
max-file: "3"
|
||||
|
||||
gitea:
|
||||
image: gitea/gitea:latest-rootless
|
||||
image: gitea/gitea:1.27.1-rootless
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- scilit
|
||||
@@ -272,18 +272,39 @@ services:
|
||||
GITEA__database__NAME: gitea
|
||||
GITEA__database__USER: gitea
|
||||
GITEA__database__PASSWD: gitea_pass_2026
|
||||
GITEA__server__DOMAIN: 123.207.9.209
|
||||
GITEA__server__DOMAIN: gitea.oncolit.gonsun.com
|
||||
GITEA__server__HTTP_PORT: 3000
|
||||
GITEA__server__ROOT_URL: http://123.207.9.209:3000
|
||||
GITEA__server__START_SSH_SERVER: "true"
|
||||
GITEA__server__SSH_DOMAIN: 123.207.9.209
|
||||
GITEA__server__ROOT_URL: https://gitea.oncolit.gonsun.com/
|
||||
GITEA__server__START_SSH_SERVER: true
|
||||
GITEA__server__SSH_DOMAIN: gitea.oncolit.gonsun.com
|
||||
GITEA__server__SSH_PORT: 2222
|
||||
GITEA__server__SSH_LISTEN_PORT: 22
|
||||
ports:
|
||||
- "3000:3000"
|
||||
- "2222:22"
|
||||
- 3000:3000
|
||||
- 2222:22
|
||||
|
||||
|
||||
caddy:
|
||||
image: caddy:2-alpine
|
||||
networks:
|
||||
- scilit
|
||||
ports:
|
||||
- "80:80"
|
||||
- "443:443"
|
||||
volumes:
|
||||
- ./Caddyfile:/etc/caddy/Caddyfile:ro
|
||||
- caddy_data:/data
|
||||
- caddy_config:/config
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: "json-file"
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
volumes:
|
||||
caddy_data:
|
||||
caddy_config:
|
||||
pgdata:
|
||||
redisdata:
|
||||
esdata:
|
||||
|
||||
+1296
-7
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,277 @@
|
||||
# 部署运维方案:生产工程化(一期 + 二期)
|
||||
|
||||
> **日期:** 2026-08-08(v17 更新于 2026-08-10)
|
||||
> **版本:** v17.0(当前) **状态:** 一期 3 项已执行(磁盘/备份/TLS)+ 构建加速(Dockerfile 卫生)+ gitea TLS,其余待执行
|
||||
> **关联:** [docs/10-生产部署文档.md](10-生产部署文档.md)、[docs/11-20260717部署事故分析.md](11-20260717部署事故分析.md)、[docs/12-部署实际操作记录.md](12-部署实际操作记录.md)、[docs/17-生产数据迁移.md](17-生产数据迁移.md)(24 迁移专项,2026-08-09)
|
||||
>
|
||||
> **版本记录:**
|
||||
> - **v17** — 2026-08-10:§7 补「服务器本地改动必须回流 gitea」环节(TLS/Caddy 改动从未提交,git-ify checkout 会丢 HTTPS——回流后服务器 git pull 才安全,与 §2 闭环)
|
||||
> - **v16** — 2026-08-10:§3 补批量迁移执行序 + destructive ordering(批次 1 日期收窄须与发代码同窗口);§7 补 gitea TLS 落地;§6 补构建加速落地(腾讯 pip/npm 源 + BuildKit,详见记忆 docker_build_optimization)
|
||||
> - **v1** — 初稿(生产工程化框架,一期 + 二期)
|
||||
> - **v2** — 吸收三档补强:破坏性迁移硬判定、backup.sh 上机查清、扩对盘、compose 范围澄清、挂卷后 logrotate、pre-deploy 快照留存、registry token 轮换、alembic head 采集、增强级
|
||||
> - **v3** — 二次反查:`.dockerignore` 脱离 git、backup.sh 连库路径两个真 bug + frontend healthcheck/双 tag/治理策略打架/工作区检查/判定扫描时机/image 写法六个设计漏洞
|
||||
> - **v4** — 吸收 9 条遗漏:worker 优雅停机、回滚复用 rollback.sh、alembic 采集命令、pull 限定服务、index.html 缓存、pgvector 升 major、并发部署锁、dump 清理排序、worker 健康检查、BuildKit secret
|
||||
> - **v5** — 固化 v3 未落点 + 10 条遗漏:§5 改 count 制、FRONTEND_TAG/前端 healthcheck/双 tag 提升主体、versions.log 双 tag schema、破坏性判定统一为 log、工作区检查入 §2、真 bug 定修复(#5/#6)、引用统一、act_runner Docker 能力、alembic_version 容错、13 条规则内联;**磁盘扩容已完成 100G**
|
||||
> - **v6** — 吸收 10 条遗漏:`--no-deps` 绕过 migrate 依赖(需显式验 migrate 退出码)、predeploy 快照全量无排除、logrotate copytruncate、.env/config 回滚耦合、compose 服务名上机 #7、自动回滚数据源 = 最近 predeploy、registry 盘余量、CI/手动共享 flock、/healthz 端点、4 workers 语义澄清
|
||||
> - **v7** — 吸收 10 条遗漏:**frontend healthcheck 改 busybox wget(nginx:alpine 无 curl,真 bug 修正)**、二期 image 改 registry 全地址固化、growpart 补步、公网 HTTP registry 凭证嗅探、回滚边界(未完成=清理/部分完成=回滚)、migrate成功+backend失败子场景、恢复演练双备份、alembic before/after 顺序、rollback 追 log、§3 加注
|
||||
> - **v8** — 吸收 25 条(P1-P15 + E1-E10):**migrate 验证改 `docker compose run --rm` 前台阻塞**(P1 up -d 异步下 ExitCode 误判 0 + P2 restart:no 二次不重跑,合并修法)、P3 predeploy 快照迁移窗口局限、P4 前端 VITE build 期固化、P5 采集命令免密码(run alembic current)、P6 daemon.json 改需重启 docker、P7 restart 已存在确认、P8 backend 无卷已查/未来加卷防线、P9 日志轮转二选一、**P10 TLS 新小节**、P11 备份推 COS 异地、P12 destructive 判定细化、P13 资源 limits、P14 冷启动 runbook、P15 alembic 多 head、E1 predeploy 清理保护、E2 rollback 双 tag 原子、E3 CONCURRENTLY/pgbouncer 迁移限制、E4 worker grace 统一 stop_grace_period、E5 rollback 跳 rollback 事件行、E6 外部反代、E7 TZ 决策、E8 RPO、E9 worker 任务幂等、E10 nginx.conf 改需 rebuild
|
||||
> - **v9** — 吸收 8 条(N1-N8):**N1(最要紧)失败自动回滚数据丢失洞——`.pending-deploy` 标记机制**(versions.log 只在成功后才写,失败处理器读它必读到"上一次成功"的 destructive=否 → 朴素换 tag → 破坏性迁移已落地 + 旧代码丢数据;改为 migrate 前写 `.pending-deploy` 含 sha/destructive/predeploy 快照路径,失败处理器读标记,成功后才挪进 versions.log)、N2 deploy.sh 合成单一序列(migrate 插 tag↔up 之间)、N3 versions.log 采集改 run --rm、N4 N1 同根因单列、N5 冷启动门对无 healthcheck 服务改判 running、N6 worker 健康门控落到可执行方案、N7 备份路径统一、N8 跑迁移前显式 export BACKEND_TAG
|
||||
> - **v10** — 吸收 8 条(M1-M4 + V5-V7):**M1 nginx /api proxy_pass 必须写 compose 服务名 `backend:8000`**(写 localhost 会跨容器调自己必失败;实测 nginx.conf:83 已正确,文档固化防改错);**M2 worker/migrate 显式用 `${BACKEND_TAG}`**;**M3 首次部署无 prev 的 destructive 判定小注**;**M4 worker/migrate 只写 `image:` 不写 `build:`**(复用预构建镜像);**V5(真实)worker healthcheck 改 python 探活**(worker 镜像无 redis-cli,N6 的 redis-cli 探活会 not found → 永远 unhealthy → 门控卡死);**V6 内置验证轮询等健康**(up 后 start_period 内勿立刻 curl);**V7 迁移期间无并发改口 run --rm 阻塞保证**
|
||||
> - **v11** — 吸收 3 条 + 2 阻塞项重申:**①gitea_data 卷备份缺口**(含 git 仓库 + 二期 registry blobs,§4 只 pg_dump 漏了它——busybox tar + COS 异地,一期标注二期前补);**②破坏性回滚数据回退落到命令形态**(优先 `run --rm backend alembic downgrade <上一 revision>` 确定性脚本化,predeploy dump 兜底,写进 rollback.sh);**③后端日志落盘提前到一期 §8**(镜像重建 json-file 日志即丢、查日志是日常刚需,与北极星直接相关;一期对齐前端 nginx 挂卷 + logrotate copytruncate + P9 二选一,二期 §4 只留 Loki/Prometheus 高级归集);阻塞级 #1 backup.sh、#8 TLS 维持不变
|
||||
> - **v12** — 吸收 5 条(A-E):**A(真实会踩)§8 backend 日志卷激活 P8 权限坑**——backend 是 `USER scilit` 非 root,挂 root 属主主机目录写不进 → 首跑即 PermissionError(nginx root 跑无此问题);按 §6 P8 结论 chown/固定 UID+卷初始化;**B(概念陷阱)破坏性回滚主路径改为 predeploy dump,downgrade 降级**——alembic downgrade 只反向 schema 不恢复数据(DROP COLUMN 要么 no-op 要么 NotImplementedError,被删列数据回不来),主路径 = pg_restore 快照;**C frontend 一期构建机制写清**(compose 含 frontend build + Dockerfile.prod,deploy.sh build 覆盖前后端);**D .pending-deploy 落盘位置明确**(部署目录持久路径、gitignore、别放 /tmp);**E gitea_data tar 备份加数量保留 N=3**;阻塞级 #1/#8 维持不变
|
||||
> - **v13** — 吸收 5 条(F-J):**F destructive 判定基线钉死**——`<prev>` = `git pull` 前的本地 HEAD,deploy.sh 在 pull 前 `PREV_SHA=$(git rev-parse HEAD)` 采集(此刻 HEAD = 服务器当前生产版本),`<sha>` = pull 后新 HEAD;严禁写成 pull 后的 `HEAD~1..HEAD`(一次 pull 多 commit 迁移会漏判/错判);**G deploy.sh 开头 `set -a; source <部署目录>/.env`**——`${PG_PASSWORD}`/`${REDIS_PASSWORD}` 用于 psql 备选采集、worker 探活,不 source 则变量为空 → 连不上/密码错;**H predeploy 用 `pg_dump -Fc`**——pg_restore 只吃 custom 格式,plain 只能 psql -f 恢复;-Fc 兼容 + 大库并行恢复 -j;**I deploy.sh 开头检测遗留 `.pending-deploy`**——上次中途崩溃(如重启)残留,先打印"上一次部署异常退出,请确认状态"再继续,不静默覆盖;**J 宿主机建 `/etc/logrotate.d/scilit-backend`**——backend + nginx 两路径同块、`copytruncate`,cron.daily 自动轮转 → **v14** — 吸收 2 条(K-L):**K 迁移 run --rm 加 `--no-deps`**(backend 的 depends_on 含 migrate 服务,`docker compose run` 默认会先拉起 migrate 依赖再跑 → migrate 服务先跑一遍 upgrade head、紧接显式那条又跑一遍——虽 alembic 幂等不报错,但迁移跑两次、与"迁移统一由 run --rm 做"的立意自相矛盾;全部 run 命令加 `--no-deps`,postgres 由冷启动健康门保证已 healthy);**L 破坏性回滚 pg_restore 补覆盖方式**(H 改 `-Fc` 后直接 `pg_restore -d scilit` 会因对象已存在报错——明确 `pg_restore --clean --if-exists -d scilit <快照>` 先清后恢复,或临时库恢复再 rename)
|
||||
> - **v15** — 2026-08-09 首轮执行 + 一次生产事故复盘:**✅ §0 磁盘扩容全部落地**(growpart + **xfs_growfs**——文件系统是 xfs 不是 ext4,resize2fs 会报 Bad magic number,§0 正文已改);**✅ §4 backup 落地**(服务器 `/root/scilit/scripts/backup.sh`,compose-exec pg_dump -Fc 机制,crontab `0 3 * * *`,手动验证 2.5GB/302 TOC/46 表);**✅ §7 TLS 走②前置 Caddy 落地**(frontend 宿主端口 80→8080、容器内仍 80,Caddy 占 80/443 自动证书,证书经 tls-alpn-01 签发成功,80→308 跳转,PUBLIC_BASE_URL/CORS_ORIGINS 已同步 https);**⚠️ 事故复盘(真坑,补进 §2/§3)**:手动 `docker compose up -d frontend` **未加 `--no-deps`** → 拉起 frontend→backend→migrate 依赖链 → migrate 以**旧镜像**跑 `alembic upgrade head` 报 `Can't locate revision '6b662a8c5235'`(**生产镜像 2-3 周未更新、落后于 DB schema**,DB head 6b662a8c5235 旧镜像不认识)→ backend/frontend 全部 Created 未启动、应用中断。恢复:恢复旧 backend 镜像 + `up -d --no-deps --no-build backend frontend`。**结论固化**:任何 `up/run` 动应用容器必须 `--no-deps`(本方案已写,执行时务必照做);**生产镜像落后于 DB schema 是部署事故隐患**——当前运行镜像(backend a6d197830cc2/frontend 95f910a68bd9,7月中)比 DB(head 6b662a8c5235)旧,真正部署新代码前需先对齐
|
||||
|
||||
## 北极星定位
|
||||
|
||||
**核心诉求 = 方便运维**——日常更新/部署更快、更稳、可回滚、少踩坑、可观测、可恢复。
|
||||
|
||||
**关键认知:**
|
||||
- 易运维的关键是**生产侧可复现、可回滚、可观测、可恢复**,不是"本地 = 生产"
|
||||
- 宿主机层(Windows vs 腾讯云 CVM)不可能完全一致;容器化保证**容器层一致**(同镜像 → 依赖 / schema / 迁移 / 构建产物一致)
|
||||
- 驱动痛点的根源是**手工多步部署脆弱**(docker cp 不持久、`__pycache__`、worker 不重启、assets 污染、迁移文件、无版本回滚)
|
||||
- **生产磁盘允许扩容**(腾讯云 EBS 可在线扩容,非破坏性)——CI + Registry 的磁盘约束随之解除
|
||||
|
||||
**总体形态**:
|
||||
- **一期(基座)**:不依赖 CI,立刻可用——版本化 + 脚本化 + 迁移安全 + 备份演练 + 镜像治理 + Dockerfile 卫生
|
||||
- **二期(自动化)**:Gitea Actions + Container Registry 全自动 CI/CD + 可观测 + feature flag
|
||||
- 一期是二期的**前置条件**(版本化、健康门控、镜像治理必须先有),二者衔接不冲突
|
||||
|
||||
---
|
||||
|
||||
## 一期:生产工程化基座(独立可落地)
|
||||
|
||||
### 0. 磁盘扩容(✅ 已全部落地 2026-08-09)
|
||||
- **2026-08-08 云盘从 20G 扩至 100G**(腾讯云 EBS);**2026-08-09 补做分区 + 文件系统扩容**:`growpart /dev/vda 1 && xfs_growfs /` → 生效 100G,可用 25G、76%
|
||||
- **⚠️ 文件系统是 xfs,不是 ext4(真坑,v15 修正)**:`df -h` 未生效时完整三步 = ①控制台扩 → ②`growpart /dev/<盘> <分区号>` 扩分区 → ③**`xfs_growfs /`** 扩文件系统。**绝不能写 `resize2fs`(ext4 专用,xfs 上直接报 `Bad magic number in super-block`)**——xfs 用 `xfs_growfs`,且 xfs 不支持缩容。先 `lsblk -f` 确认 FSTYPE 再选工具
|
||||
- 保留认知(扩容时已确认目标盘):pgdata/redisdata/esdata/miniodata/gitea_data(registry 复用 gitea_data)是**不同卷、可能在不同挂载点**,CI 缓存(二期)在 runner 本地——已扩 pgdata 所在吃紧盘
|
||||
- 容量依据:pgdata 卷增长 + 多版本镜像(N=5,后端 ~500M×5)+ CI 构建缓存(二期)+ registry 存储(二期,复用 gitea_data 卷)+ 日志
|
||||
|
||||
### 1. 版本化镜像标签(镜像即版本)
|
||||
- 每次构建打 **git-sha 标签**:`docker tag scilit/backend:latest scilit/backend:<git_short_sha>`(前端同理)
|
||||
- **compose 用环境变量插值引用 git-sha(后端 + 前端双变量,v5 固化到主体)**:`image: scilit/backend:${BACKEND_TAG:-latest}`、`image: scilit/frontend:${FRONTEND_TAG:-latest}`,deploy.sh 同时传 `BACKEND_TAG=<sha> FRONTEND_TAG=<sha>`——compose 文件稳定、git pull 不冲突,sha 只在运行时传入。不用 `latest` 当生产版本(`latest` 不指向确定提交,仅便捷别名)
|
||||
- 保留最近 N 个旧 tag(磁盘允许下 N=5–10);**回滚 = 换 tag 重启**,秒级
|
||||
- 前置:给 backend/worker/**migrate**/frontend 在 `docker-compose.prod.yml` 显式 `image:`(当前由 compose project 名生成,基线不稳定);**worker/migrate 与 backend 共用同一镜像 + 标签,显式用 `${BACKEND_TAG}`(M2)**——backend/worker/migrate 三服务 `image: scilit/backend:${BACKEND_TAG:-latest}`、frontend `image: scilit/frontend:${FRONTEND_TAG:-latest}`,**跑迁移的代码版本必须 = 生产版本**(否则 migrate 用的是旧镜像,迁移与代码不同步);**worker/migrate 只写 `image:`、不写 `build:`(M4)**——复用 deploy.sh 预构建的版本镜像,移除 `build: ./backend`,避免"声明了 build 但部署时不 build"的矛盾语义(backend 保留 build 作为构建源)
|
||||
- **前端缓存策略配套(换 frontend 镜像后浏览器缓存 404)**:旧浏览器缓存的 `index.html` 会去请求已不存在的旧 hash 资源 → 404。nginx 对 `index.html` 设 `Cache-Control: no-cache`,对 `assets/*`(哈希文件名)长缓存 `immutable`——改 `frontend/nginx.conf`;**⚠️ nginx.conf 是 COPY 进镜像的(E10)**——改它必须 rebuild 前端镜像重新部署,`docker cp` 进容器不持久、重启即丢
|
||||
- **⚠️ nginx /api 的 proxy_pass 必须写 compose 服务名 `backend:8000`(M1,文档固化防改错)**:实测 `frontend/nginx.conf:83` 已正确写 `set $backend_upstream http://backend:8000;`——**写 `localhost:8000` 会跨容器调自己、必然失败**(frontend 容器内 8000 无服务)。约束固化:proxy_pass 目标永远用** compose 服务名 + 端口**(backend:8000),不用 localhost/127.0.0.1;nginx 侧 resolver 动态解析(nginx.conf 已配 `resolver 127.0.0.11`)配合 backend 容器重建后 IP 变化
|
||||
- **frontend healthcheck + 双 tag 原子回滚(v5 提升到主体)**:frontend 需自身 healthcheck,不能只靠 `depends_on: backend: service_started`;**⚠️ nginx:alpine 默认不含 curl(v7 真 bug 修正)**——healthcheck 用 **busybox `wget`**(alpine 自带):`wget -q -O- http://localhost/ || exit 1`,或 Dockerfile 另装 curl;**回滚 = backend+frontend 双 tag 原子切换**——两者同一次部署、同一条 versions.log 记录,绝不允许只回一个导致前后端版本错配
|
||||
|
||||
### 2. 脚本化部署(消灭 13 条规则的坑)
|
||||
> **"13 条规则"来源内联(v5)**:docs/10-生产部署文档.md §12 的 13 条严格部署规则(记忆 production_deployment_rules.md)——本方案将其固化为脚本,此处不重复罗列,执行时以脚本为准
|
||||
- **compose 范围澄清(阻塞级)**:`up -d --no-deps backend worker frontend` 意味着 gitea/postgres/redis/es/minio 必须已在跑——**先确认这些服务与 backend 在同一 `docker-compose.prod.yml`**(`docker compose ps` 实查),部署前基础设施已在跑,避免漏起或误动
|
||||
- **冷启动 vs 热部署(P14)**:`up -d --no-deps` 假设基础设施已在跑,**服务器重启后全栈 down 时直接跑 deploy.sh 会连不上 db**——deploy.sh 顶部加**基础设施健康门**:postgres/redis/es/minio 状态异常即中止并提示先冷启动;**⚠️ 门控判定按"有 healthcheck 判 healthy、无 healthcheck 判 running"(N5——不能一刀切要求 healthy,否则对没配 healthcheck 的服务门控永远失败、deploy 永远跑不了)**;注:当前 prod compose 四服务均已配 healthcheck(pg_isready / redis-cli ping / es curl / minio curl),但脚本仍按 running 兜底、防未来新服务漏配;冷启动 runbook(写进 docs/10):`docker compose up -d postgres redis es minio gitea` → 等 `docker compose ps` 达门控标准 → 再走 deploy.sh
|
||||
- **`deploy/deploy.sh`(v9 合成单一序列,N2——migrate 编进有序步骤,照做不漏步)**:**脚本开头先 `set -a; source <部署目录>/.env; set +a`(G——`${PG_PASSWORD}`/`${REDIS_PASSWORD}` 在 psql 备选采集、worker 探活里用到,不 source 则变量为空 → psql 连不上/探活密码错;`.env` 不进 git,靠运行时加载)** → ①**工作区干净校验**(`git status --porcelain` 为空,否则 hotfix 残留导致 pull 冲突,冲突即中止)→ **①.5 采集 `PREV_SHA=$(git rev-parse HEAD)`(F——destructive 判定基线,必须在本步 pull 之前,见 §3)** → ②`git pull` → ③**预部署 pg_dump 快照(安全网)** → ④build + 打 git-sha 标签 + **`export BACKEND_TAG=<sha> FRONTEND_TAG=<sha>`**(给后续 `run --rm`/`up` 用,N8;**frontend 镜像一期来源写清(C)**:compose 已含 `frontend: build: {context: ./frontend, dockerfile: Dockerfile.prod}`([frontend/Dockerfile.prod](frontend/Dockerfile.prod))——deploy.sh 的 build 步对 backend/frontend 都 `docker compose build`,一期无 CI 也不用手动 `docker build -f`) → ⑤**采 before alembic head → 写 `.pending-deploy` 标记(N1,见下条)** → ⑥**跑迁移** `docker compose run --no-deps --rm backend alembic upgrade head`(K——**必须 `--no-deps`**:backend 的 depends_on 含 migrate 服务,`run` 默认先拉起 migrate 依赖再跑 → migrate 服务先跑一遍 upgrade、紧接这条又跑一遍,迁移跑两次且与"迁移统一由 run --rm 做"自相矛盾;postgres 由冷启动健康门保证已 healthy,`--no-deps` 不会连不上;前置:postgres healthy;**退出码非 0 即中止**,见下条)**→ 成功后采 after alembic head** → ⑦`up -d --no-deps backend worker frontend`(显式指定,不动基础设施;migrate 的 depends_on 门控作双保险) → ⑧**内置验证——轮询等健康(V6)**:up -d 后新 backend 仍在 start_period(healthcheck 未过),**立刻 curl 会命中启动中、误判失败**——**轮询 `/health` 直到 healthy 或超时(如 60s×5s)**,再验前端 200 → ⑨**全部成功后才把 `.pending-deploy` 挪进 versions.log**(追加一行 + 删标记) → ⑩镜像治理清理。**失败处理分两段(v7 边界 + v9 读标记修正)**:**①部署未完成(③-⑥之间失败,迁移未跑)** → 容器还是旧的、**无需数据回滚**,只清理失败中间态(临时镜像/标签)即可;**②部署部分完成(迁移已跑/容器已切)** → 调用 `rollback.sh`——**读本次 `.pending-deploy` 而非 versions.log(N1,见下条)**;**回滚数据源(v6 补)**:需数据回退时**优先用 `.pending-deploy` 里记的本次 predeploy 快照**(部署前最新状态),**绝不用更早的 dump**
|
||||
- **⚠️ `.pending-deploy` 标记(N1 最要紧——失败自动回滚的数据丢失洞修复)**:versions.log **只在部署成功后才写**——若某次部署跑了**破坏性迁移(已成功落地)**、紧接着 backend up 失败,失败处理器调 rollback.sh 时读 versions.log,读到的必然是**上一次成功部署**的 `destructive=否` → **朴素换 tag、不回退数据** → 破坏性迁移已落地 + 旧代码 = 数据/代码不匹配、**丢数据**。**修复:deploy.sh 在 ③快照后、⑥迁移前写 `.pending-deploy`**,内容含 `<backend_sha> <frontend_sha> <destructive: 是/否> <本次 predeploy 快照路径>`(destructive 由 §3 的 git diff 判定此时即算出)——**失败处理器②只读 `.pending-deploy`,绝不读 versions.log**;**N4(N1 同根因单列)**:任何"部署失败时读 versions.log 判定本次 destructive"的做法都是这个洞——versions.log 记录的是上次成功,覆盖不了本次失败,统一只认 `.pending-deploy`。**⑨全部成功后才把它挪进 versions.log**(追加 + 删除标记);失败路径回滚完成后同样清除标记;**⚠️ 落盘位置(D)**:`.pending-deploy` 是 deploy.sh 在主机写的**运行时标记**——必须落在**持久主机路径**,即部署目录内稳定位置 `<deploy_dir>/.pending-deploy`(**别放 /tmp**,重启即清;**别放会被 `git clean`/hotfix 清理或与 `git pull` 冲突的位置**),且该文件加入 **`.gitignore`**(运行时状态不进 git);**⚠️ 遗留标记检测(I)**:deploy.sh 开头(source .env / 工作区校验前)先查 `<deploy_dir>/.pending-deploy` 是否存在——存在说明**上一次部署中途崩溃(如服务器重启)未正常收尾**,deploy.sh **不得静默覆盖**:先打印"⚠️ 上一次部署异常退出,残留 `.pending-deploy`,请先确认当前状态(容器/迁移/versions.log)再继续",人工确认后清理标记或由脚本继续
|
||||
- **强制重建**:`up -d` 默认镜像 tag 变才重建,若 `${BACKEND_TAG:-latest}` 解析出的 latest 与旧容器一致会**不重建容器**("改了代码没更新"经典坑)——deploy.sh **始终传 BACKEND_TAG=<新sha>** 或加 `--force-recreate`
|
||||
- **migrate 时机 + 显式验证(v8 关键重写——P1+P2 合并修法)**:`up -d --no-deps` 会**跳过全部 depends_on 检查**,只靠时间顺序无法保证 backend 等 migrate 退出。且**原 v6 的"`up -d migrate` + `docker inspect ExitCode`"方案本身有两个真 bug**:**P1** `up -d` 异步返回,容器仍在 running 时 `inspect .State.ExitCode` 恒为 0(running 状态 ExitCode 无意义)→ **migrate 还没跑完就被误判成功**;**P2** migrate `restart: "no"`,首次跑完即 exited,**二次部署时 `up -d migrate` 对已 exited 且配置未变的容器不重跑** → **迁移静默跳过**。**合并修法:迁移统一用 `docker compose run --no-deps --rm backend alembic upgrade head`(K——必加 `--no-deps`,见上条)**——①前台阻塞到容器退出,**退出码即真值**(无 P1 误判);②每次都是全新容器**确定性重跑**(无 P2 跳过);③容器内走 `DATABASE_URL` 自带密码(顺带免 P5 的 PGPASSWORD 问题);④`--no-deps` 不拉起 migrate 依赖,避免迁移跑两次。**⚠️ 跑 `run --no-deps --rm backend` 前必须先 `export BACKEND_TAG=<新sha>`(N8)**——`run --no-deps --rm` 用的是 compose 的 `image:` 字段,`${BACKEND_TAG:-latest}` 未设则解析成 latest → **拿旧镜像跑迁移**(旧迁移、对不上新代码);deploy.sh 在 ④build/tag 时已 export(见上条),但脚本内 ⑥迁移步骤前显式再确认一次(幂等,防手改脚本漏掉)。前置:postgres 必须 healthy(冷启动门已保证,见上条 P14)。**退出码非 0 即中止部署**(这同时落实了 §3"失败即中止/迁移期间无并发"的真正保证点)→ 确认退出 0 后再 `up -d --no-deps backend worker frontend`。migrate 服务保留在 compose(作为声明式迁移入口 + 非 `--no-deps` 路径的 depends_on 门控),但**脚本判定只认 `run --no-deps --rm` 的退出码**
|
||||
- **`deploy/rollback.sh`(v9 双入口修正,N1 配套)**:换旧 tag + 重启;**两种入口读不同来源——①手动回滚(人主动跑、目标是任意历史版本)**:旧 sha + destructive 从 **versions.log** 读(**跳过 rollback 事件行,E5**——取最近的非 rollback deploy 行,否则会回滚到"上一次回滚"、甚至反复回滚循环);**②部署失败自动回滚(deploy.sh 失败处理②调)**:读**本次 `.pending-deploy`** 的 sha + destructive + 快照路径——**绝不用 versions.log**(那是上一次成功的判定,正是 N1 的洞);**先判迁移**(本次 sha 是否带新迁移,决定是否先数据回退再换 tag);**破坏性数据回退落到命令形态(v12 概念修正——downgrade 不恢复数据)**:**数据恢复主路径 = `pg_restore` 本次 predeploy 快照**(数据 + schema 一起回);**⚠️ 覆盖现有库的方式(L——H 改 `-Fc` 后的收尾)**:目标库已有旧 schema/数据,直接 `pg_restore -d scilit <快照>` 会因**对象已存在**报错——必须**先清后恢复**:`pg_restore --clean --if-exists --no-owner -d scilit <快照>`(`--clean` 先 DROP 已存在对象、`--if-exists` 缺对象不报错、`--no-owner` 免属主匹配),或更稳妥的**临时库恢复再 rename**(起临时库 `createdb scilit_restore_tmp` → 恢复 → 校验 count → 停 backend → `ALTER DATABASE scilit RENAME TO scilit_old; ALTER DATABASE scilit_restore_tmp RENAME TO scilit` → 起 backend;rename 方式迁移窗口更短,适合大库);**⚠️ alembic `downgrade()` 只反向 schema、不恢复数据**——DROP COLUMN 的迁移 downgrade 要么 no-op 要么 NotImplementedError,**被删列的数据永远回不来**,若优先 downgrade 会"以为安全其实不安全"。downgrade 仅用于**可逆的非破坏性 schema 调整**这类少见场景(且同样丢该 schema 内数据);downgrade 目标 revision(如用)= `.pending-deploy`/versions.log 里的 **before alembic head**;**回滚动作也追加一条 versions.log(标记 rollback 事件,v7 补)**——便于审计回溯「何时部署、何时回滚、回滚到哪个 sha」;**双 tag 原子切换(E2)**:`BACKEND_TAG=<旧sha> FRONTEND_TAG=<旧sha>` **同时传**、一次性 `up -d`——绝不分两次 up(中间态前后端版本错配)
|
||||
- **`deploy/hotfix.sh`**:仅紧急单文件,**强制回流**——用后必须 git 提交 + 正式部署,杜绝"手工改与仓库不一致"重演;**末尾强制收口**:打印"docker cp 不进镜像、容器重建即丢"警告 + 提示限时完成正式部署(hotfix 属脆弱窗口)
|
||||
- **.env/config 与代码版本耦合(v6 补,回滚漏项)**:新代码可能依赖新 env 变量,回滚旧代码后 .env 仍是新的——旧代码若**缺必需变量会启动失败**。处理:①代码对新增配置**尽量给默认值/可选**(向后兼容),回滚旧代码总能启动;②`.env` 本体**仍不进 git**(含密钥),非机密配置默认值随 `docker-compose.prod.yml`/`.env.example` 走 git 版本;③回滚 = 换 tag 时**确认旧代码不依赖本次新增的必需变量**(deploy.sh 可在回滚前 diff 校验);**前端是 build 期固化(P4,与后端机制不同)**:Vue 的 `VITE_*` 变量在 `npm run build` 时写死进 dist——运行时改 compose 的 `VITE_API_BASE` **不生效**,改前端任何构建期变量 = 必须 rebuild 前端镜像重新部署(当前 compose 里 frontend 的 `VITE_API_BASE: /api/v1` 是部署时摆设,真值在构建时已固化)
|
||||
- **Worker 优雅停机(防长任务丢失)**:镜像优先会重建 worker,但 `up -d` 默认立刻 kill——worker 正跑长任务会丢/坏任务。**worker 捕获 SIGTERM 完成当前任务或重新入队(ARQ 支持 graceful shutdown),compose 设 `stop_grace_period: 60s`(E4 统一此处,删掉 deploy.sh 手动 SIGTERM——compose 重建时本来就会先 SIGTERM 再等 grace,手动发是重复机制)**;**⚠️ 重入队的前提是任务幂等(E9)**:SIGTERM 后任务回到队列重跑,若任务**非幂等**(重复执行有副作用,如重复发邮件/重复扣款/重复写重复数据)则优雅停机反而造成重复执行——部署前确认所有 ARQ 任务幂等(作业内做去重/幂等键),否则仅完成当前任务、不重入队
|
||||
- **并发部署锁**:两人/两终端同时 deploy 会抢 tag 和 versions.log。**deploy.sh/rollback.sh 顶部 `flock` 单实例锁**(`exec 9>/tmp/scilit-deploy.lock; flock -n 9`),拿不到锁即中止。一期手动风险低,二期 CI 自动部署后变硬需求——现在定习惯成本最低
|
||||
- **`deploy/versions.log`**:每次部署记录 `<时间> <backend_sha> <frontend_sha> <迁移前/后 alembic head> <镜像> <destructive: 是/否>`(v5 双 tag schema)——**后端/前端独立 tag**,供 rollback 双 tag 原子回滚读旧版本;**alembic head 由 deploy.sh 采集(宿主机无 venv,必须经容器执行)**——**顺序(v9 统一 run --rm,N3):①迁移前采 before ②`docker compose run --no-deps --rm backend alembic upgrade head` 成功(退出码 0)后采 after**(原 v7 写的是 `up -d migrate`,与 P1 改 run --rm 矛盾,统一;before 采集在写 `.pending-deploy` 前、after 在⑥迁移后),各执行 **`docker compose run --no-deps --rm backend alembic current`**(容器内走 `DATABASE_URL` 自带密码,**无需 PGPASSWORD——P5**;psql 备选 `docker compose exec -T postgres psql -U scilit -d scilit -tAc "SELECT version_num FROM alembic_version"` 在 pg_hba 非 trust 时会要密码,须加 `PGPASSWORD=${PG_PASSWORD}` 前缀)落库(一次性 migrate 容器跑完即退,无法自行回传,必须 deploy.sh 代采);**⚠️ 首次部署 alembic_version 表可能不存在(v5 容错)**:采集前先 `SELECT to_regclass('alembic_version')` 判存在,表不存在 → 记空,否则首跑即 abort;**自身轮转**——每次追加后 `tail -n 200` 截断(或配 logrotate),防高频部署下无限增长
|
||||
|
||||
### 3. 迁移安全(高频升级最易翻车点)
|
||||
- migrate 独立成服务 + `depends_on: migrate: service_completed_successfully`(已在 prod compose)
|
||||
- **向后兼容规范**:先加字段/表,不删不改旧结构;旧代码全下线后下一版再清理
|
||||
- **失败即中止**:migrate 失败 → backend/worker 不启动,不替换容器;**⚠️ 加注(v7 起)**:在手动 `up -d --no-deps` 流程下,这**并非 compose 自动保证**——`--no-deps` 跳过 depends_on,真正保证在 §2 的「`run --rm` 前台跑迁移、退出码非 0 即中止」步骤(v8 起判定方式见 §2),勿误读为 compose 自动行为
|
||||
- **迁移期间无并发(V7 改口,与 §2 一致)**:**`run --rm` 前台阻塞保证** migrate 完成(exit 0)后才起 backend/worker(手动 `--no-deps` 路径下 compose 的 depends_on 顺序**不生效**,同 §2 N2/N3——真正保证在 §2 的 run --rm 前台阻塞 + 退出码判定),避免"新迁移 + 旧代码"并发跑在旧 schema 上;首次/失败路径也要确认不出现并发(部署窗口内 backend 保持旧版直至 migrate 通过)
|
||||
- **⚠️ 2026-08-09 事故复盘(真坑,v15 补)**:手动 `docker compose up -d --no-build frontend` **没带 `--no-deps`** → compose 按依赖链拉起 frontend→backend→migrate;migrate 以 `scilit-migrate:latest`(**旧镜像,2-3 周未更新**)跑 `alembic upgrade head` → **`Can't locate revision '6b662a8c5235'`**——**生产 DB schema(head 6b662a8c5235)比运行中镜像认识的 head 新**,旧镜像的 alembic/versions 里没有该 revision → migrate 退出 255 → backend(depends_on migrate service_completed_successfully)与 frontend 全部 **Created 未启动、应用中断**。**恢复**:`docker tag <旧backend镜像> scilit-backend:latest` + `docker compose up -d --no-deps --no-build backend frontend`。**教训固化**:①任何动应用容器的 `up/run` **必须 `--no-deps`**(§2 已写死,执行时照做,别省略);②**生产镜像落后于 DB schema 是隐患**——当前运行镜像(backend a6d197830cc2 / frontend 95f910a68bd9,2026-07-17 构建)比 DB head 旧,`upgrade head` 在这种状态下**必然失败**;真正部署新代码前需先把镜像更新到与 DB 对齐的版本(§1 版本化镜像正是解药);③`up -d <服务>` 的依赖链是 **frontend→backend→migrate**,误触发 migrate 的代价是整条链全停——冷启动/单独起某服务一律用 `--no-deps`
|
||||
- **迁移回滚 runbook(写进 docs/10)**:
|
||||
- **明文约定**:只要坚持"向后兼容、只增不删",回滚(换旧 tag)就是安全的——旧代码对新加的列/表可忽略
|
||||
- **破坏性迁移硬判定(v5 统一为 deploy 时落 log,替代扫工作区)**:破坏性迁移(删列/改类型/重建表)文件统一命名 `destructive_*.py`(或迁移文件头部醒目 `# DESTRUCTIVE` 注释)**作双保险**;**主判定改为 deploy.sh 部署时基于 `git diff <prev>..<sha> -- alembic/versions/` 判定,把"是否 destructive"写进 versions.log**;**⚠️ `<prev>` 基线钉死(F)**:`<prev>` = **`git pull` 前的本地 HEAD**——deploy.sh 在 ①工作区校验后、②`git pull` 前采 `PREV_SHA=$(git rev-parse HEAD)`(此刻 HEAD = 服务器当前生产版本),`<sha>` = pull 后新 HEAD,`git diff PREV_SHA..<sha>` = 本次部署真正引入的迁移;**严禁写成 pull 后的 `HEAD~1..HEAD`**——一次 pull 常带入多个 commit 的迁移,`HEAD~1` 不是上次生产版本,diff 会漏判/错判 destructive;**⚠️ 判定规则必须细化,不能笼统"检测 drop/alter"(P12——过粗会让每次 `ADD COLUMN` 都误触发、强制数据回退)**:判 destructive **只看**真正破坏性操作——`DROP TABLE / DROP COLUMN / DROP INDEX`、`ALTER COLUMN TYPE`(类型变更)、`RENAME`(表/列/索引重命名)、`ALTER COLUMN SET/DROP NOT NULL` 收紧、重建表(create_table 后 drop 原表)、破坏性数据变更(批量 UPDATE/DELETE);**明确不计入(良性)**:`ADD COLUMN`、新建表、`CREATE INDEX`(非 CONCURRENTLY)、`ALTER COLUMN SET DEFAULT / DROP DEFAULT`、加约束——向后兼容,标非破坏性——`rollback.sh` 回滚时**读 log 的 destructive 字段**而非扫当前工作区(扫工作区会被后续版本删除/改名骗过 → 漏判破坏性迁移 → 以为安全回滚其实丢数据)。标记 destructive → 强制先数据回退(**主路径 pg_restore 本次 predeploy 快照,downgrade 仅做 schema 反向不恢复数据——v12 概念修正,命令形态见 §2 rollback.sh**)再换旧 tag;无 → 直接换 tag 安全;**首次部署无 prev(M3,小注)**:`git diff <prev>..<sha>` 无基线 → **改为直接对当前 `alembic/versions/` 目录做同样的 keyword 检测**——初始迁移多为 CREATE 建表,实测判非破坏性、风险低,不需特殊流程,小注记录即可
|
||||
- **代码回滚 ≠ 迁移回滚**:Alembic 单向递增,回滚代码一般不回退迁移;含破坏性迁移时才触发数据回退流程
|
||||
- **migrate 成功 + backend 失败子场景(v7 补)**:此时数据已是新 schema(且向后兼容规范下**只增不删**)——回滚只需换 tag、**不需数据回退**,与 rollback.sh 读 destructive=非破坏性(直接换 tag)一致
|
||||
- **Postgres 大版本锁定**:`pgvector:pg16` 的 major 与数据卷**强绑定**;**升 major 必须 pg_dump/restore 迁数据**,绝不直接 `up -d` 换镜像(否则数据卷不兼容起不来)——此条写进 docs/10 显眼位置
|
||||
- **pgvector 升 major 的特殊性(restore 前必查)**:dump/restore 恢复 vector 数据时,**目标库必须先 `CREATE EXTENSION vector`**,且扩展版本与目标 pgvector 镜像匹配;vector 索引(ivfflat/hnsw)恢复依赖扩展存在,先建扩展再恢复,否则首次升 major 必踩
|
||||
- **alembic 多 head 必须提前拦住(P15)**:并行 PR / 单人多分支各自新增迁移、同 down_revision → `alembic heads` 返回多个 → `upgrade head` **直接报错中止、不应用任何迁移**。防线:①CI 加一步 `alembic heads` 校验,>1 即失败;②deploy.sh 迁移前 `docker compose run --no-deps --rm backend alembic heads` 预检,多 head 即中止;③多人协作约定:合并前先 `alembic merge` 或串行 rebase 迁移(一人一个 base)
|
||||
- **迁移执行受限操作(E3)**:`CREATE INDEX CONCURRENTLY` **不能跑在事务块内**——alembic 默认把迁移包在事务里,需并发建索引用 `with op.get_context().autocommit_block():`;未来若上 pgbouncer 事务池,长事务迁移会被池限制影响(迁移建议直连 postgres 服务、绕过池)
|
||||
- **批量迁移执行序 + destructive ordering(v16 补,2026-08-10,专项见 docs/17)**:生产 DB 落后本地多个迁移时,**不能整条链一次 `upgrade head`**,须分批执行到中间 checkpoint(`migrate_prod.sh 1|2|3|4`,见 docs/17 §4)。**关键纪律——destructive 批次不能提前单独跑**:
|
||||
- **批次 1(`1421ea169bb6` 日期 TIMESTAMPTZ→DATE)是类型收窄**——老代码读 datetime 会崩,**必须与发新代码同窗口**(迁移完几秒内新镜像接管),绝不能提前单独跑
|
||||
- **批次 2/3 纯 additive**(加列/新表/索引),老代码兼容,**可提前任意时段跑**,缩小维护窗口
|
||||
- **批次 4(`95c18ebf31e4`/`55105f0bb1d7` VARCHAR→Text + 大数据量回填)须新代码已部署后深夜跑**
|
||||
- 推荐序:**批次 2/3(提前)→ 维护窗口:build 新镜像 → 批次 1 → 发新 backend/worker/frontend → 深夜:批次 4**
|
||||
- 配套:build 加速已落地(§6 Dockerfile 卫生),迁移用 `run --no-deps --rm backend alembic upgrade <rev>`(新镜像自带新迁移,勿退回老容器 exec)
|
||||
- **统一备份脚本(✅ 已落地 2026-08-09,原阻塞级 #1 查清)**:上机确认 **crontab 原本无任何 backup 条目、backup.sh 根本没在跑**(历史 docs/10 的 `/home/scilit/backup.sh` 不存在),且仓库版 `PG_HOST=localhost` 连不上(postgres `ports: []` 无宿主端口)——**已新建 `/root/scilit/scripts/backup.sh` 并落 crontab `0 3 * * *`**,机制为 **`docker compose -f docker-compose.prod.yml exec -T postgres pg_dump`**(不经宿主端口,容器内 pg_dump 16.14);排除表 pipeline_runs/api_usage_logs;`--format=custom --no-owner --no-privileges`;备份目录 `/data/backups`;保留 30 天;**手动验证成功**:2.5GB / `pg_restore -l` 302 TOC / 46 表 / Format CUSTOM。**路径统一约定(N7)**:脚本 = **仓库 `backend/scripts/backup.sh`**(随 git 分发,服务器部署目录内执行),备份目录 = **`/data/backups`**——服务器实际路径 `/root/scilit/scripts/backup.sh` 与仓库 `backend/scripts/backup.sh` 需在正式部署时统一对齐(当前以服务器实际为准);docs/10 的 `/home/scilit/backup.sh`、`/backup` 等历史路径**全部废弃**
|
||||
- **`.env` 单点备份**:含 SMTP/JWT/API Key 全部密钥,git pull 不动它但只存服务器——**离线备份(不进 git)**,防丢密钥
|
||||
- **pre-deploy 快照 = 全量 pg_dump,与日常备份分开(v6 修正)**:§2 用 predeploy dump 当"全量安全网",但若复用排除 pipeline_runs/api_usage_logs 的 backup.sh,安全网本身就缺这两表、与"全量"定位冲突——**predeploy 必须用不带排除的完整 `pg_dump`**(含 pipeline_runs/api_usage_logs),与日常 backup.sh 分开执行;**⚠️ 用 `-Fc` custom 格式(H——pg_restore 只吃 custom 格式)**:§2 rollback.sh / §4 restore-drill 的数据回退都走 **`pg_restore`**,而 **pg_restore 要求 dump 为 `-Fc` custom 格式**——plain 文本格式(pg_dump 默认)只能 `psql -f` 恢复,pg_restore 直接拒绝;**格式钉死:predeploy 用 `pg_dump -Fc`**(custom 兼容 pg_restore,且大库可并行恢复 `-j`);日常 backup.sh 已用 `--format=custom`,核对生产实际跑的那份保持一致;**⚠️ 快照是"尽力安全网",不覆盖迁移窗口写入(P3)**:快照在**旧 backend 仍在服务时**拍的——拍完到新 backend 上线之间(migrate + 容器切换窗口)仍有业务写入,此窗口内回退会**丢这几分钟数据**。这是快照式安全网的固有局限、非 bug:长窗口靠日常 03:00 backup + RPO 预期(见 E8)兜底,部署窗口的分钟级丢失在低峰 + 短迁移下可接受——**不做"先停写再拍快照"的 drain 步骤**(单机不值当),但认知要写清
|
||||
- **pre-deploy 快照留存策略(容易漏)**:deploy.sh 每次部署前 dump 当安全网,高频部署下吃磁盘——**保留最近 N=3 个**(按数量清理,如 `ls predeploy_*.dump | sort | head -n -3 | xargs rm`),否则备份目录先爆;**⚠️ head -n -3 语义保护(E1)**:GNU `head -n -3` 是"去掉最后 3 行"——文件 ≤3 个时输出为空、`xargs rm` 无输入不执行,**不会误删但语义易读错**。deploy.sh 显式写成 `count=$(ls ... | wc -l); [ "$count" -gt 3 ] && ls ... | sort | head -n -3 | xargs -r rm`(`-r` 空输入不执行),防笔误
|
||||
- **⚠️ 清理依赖文件名可排序**:上面的 `ls | sort` 要靠文件名里嵌入**可排序的 ISO 时间戳**(`YYYYMMDD_HHMMSS`)才正确挑最旧;用别的格式(如相对时间命名)会删错文件——deploy.sh 统一命名规范
|
||||
- **补恢复演练**:`restore-drill.sh`(起临时 postgres → pg_restore → 校验 count)或文档化步骤,定期演练——否则备份等于没备;**演练覆盖两种备份(v7 补)**:①全量 predeploy(含 pipeline_runs/api_usage_logs)②日常排除 backup——**分别校验**,不能只验一种(排除表缺失/为空是否可接受要在两套上各自确认)
|
||||
- **恢复校验覆盖排除表**:backup.sh 排除了 pipeline_runs/api_usage_logs,恢复演练**校验这些表缺失/为空是可接受的**(避免"count 一致但关键排除表没恢复"的误判)
|
||||
- **备份必须异地(P11——同盘非真 DR)**:backup.sh 写 `/data/backups`(同 CVM 磁盘),**磁盘故障时备份与库俱毁,备份等于没备**。补:备份完成后自动上传**腾讯云 COS**(项目已有 `COS_SECRET_ID/KEY/BUCKET` 凭据,S3 兼容,coscli/rclone 均可)或 rsync 到另一节点;**上传失败必须告警**(备份不能静默失败);`.env` 同样纳入离线异地(已有原则)
|
||||
- **gitea_data 卷备份(v11 补——真实缺口)**:§4 只做 pg_dump,但 **gitea_data 卷(含 git 仓库 + 二期 registry 镜像 blobs)完全没进备份范围**——此卷一丢,所有仓库 + 二期镜像全没、要全部重 push。**一期先标注"此卷需单独备份",二期前补执行**:`docker run --rm -v gitea_data:/src -v /data/backups:/dst busybox tar czf /dst/gitea_data_<ts>.tar.gz -C /src .`,同样传 COS 异地;注意 gitea 容器运行中 tar 的一致性(git 仓库文件持久、轻微不一致可接受;严格则先 `docker compose stop gitea` 再 tar);registry blobs 量大,纳入异地时评估体积/频率(可低频率如每周);**保留策略(E)**:tar 备份**按数量清理**——保留最近 N=3 个(同 predeploy 的 N=3 思路),文件名嵌可排序 ISO 时间戳(同 §4 命名规范),`ls gitea_data_*.tar.gz | sort | head -n -3 | xargs -r rm`,防盘爆
|
||||
- **RPO 预期明示(E8)**:日常 backup 每日 03:00 → **最坏 RPO ≤ 24h**(backup 失败可能拖到 48h,靠告警兜底);predeploy 快照随每次部署拍 → 部署窗口内 RPO 分钟级。**需用户确认这个 RPO 是否可接受**,不可接受则加密日常备份频率(如每 6h)——先写清预期,不擅自设默认值
|
||||
|
||||
### 5. 镜像治理(扩盘后仍需防爆)
|
||||
- **应用镜像只按 count 清理(v5 修正,消除与 §1 矛盾)**:保留最近 N=5-10 个**带 tag 的版本镜像**(与 §1 一致),超出删除——**绝不按 age 清理带 tag 的应用镜像**(否则低频部署时 7 天外的保留 tag 被 age-prune 清掉 → 回滚点静默丢失)
|
||||
- **age-prune 只作用于 dangling/build cache**:`docker image prune --filter "until=168h"` 只清不带 tag 的中间层 + build cache(`-a` 仍绝不使用,避免误删构建缓存)
|
||||
- 磁盘监控:定期 `df -h`,超阈值告警
|
||||
|
||||
### 6. Dockerfile 卫生(构建加速)
|
||||
> **一期 deploy.sh 在生产机 build 的前提**:Phase 1 无 CI,镜像在 CVM 上 `docker build`——生产机必须**能拉基础镜像**(python:3.12-slim、node:20-alpine、nginx:alpine、gitea 基础镜像可达)+ **具编译能力**(psycopg2/pgvector 编译,需 build-essential/libpq-dev,backend Dockerfile 已装)。腾讯 apt/pip 镜像已配,这块已具备;**首次跑前确认**网络与源可达。
|
||||
- `backend/Dockerfile`:腾讯 apt/pip 镜像(回归历史 `docs/12` §4.2–4.5 的加速做法,当前已丢失)
|
||||
- `frontend/Dockerfile.prod`:`npm ci` + package-lock.json + npmmirror
|
||||
- **`.dockerignore` 统一补齐**(防密钥进镜像层 / 防旧字节码 / 缩构建上下文):
|
||||
- backend:`__pycache__/`、`*.pyc`、`.env`、`tests/`、`scripts/`、`data/`、`.pytest_cache/`、`*.egg-info/`(现有已含 `__pycache__`/`.env`,补齐其余)
|
||||
- frontend:补 **`.env`**(当前未排除,可能把 dev 环境变量打进镜像)、`__pycache__`、`*.pyc`;保留 `node_modules`/`dist` 排除
|
||||
- **新增 `.gitattributes`**:`* text=auto eol=lf`——防 Windows 编辑 .sh/Dockerfile 的 CRLF 在 Linux 容器内报错
|
||||
- **构建期密钥防进镜像层(防未来踩坑)**:若 backend 未来需私有 pip 源 token,别用 build ARG 写进镜像层——用 BuildKit `--mount=type=secret`。当前腾讯公开源不需要,但一句话防未来私有源踩坑
|
||||
- **非 root 容器 + 卷权限(P8,已查实当前无卷、须防未来)**:backend 以 `USER scilit` 跑(backend/Dockerfile:29),**当前 backend/worker 容器没有任何卷挂载**(文件存储走 MinIO/COS,`docker compose config` 实查确认)→ **现无此问题**;但**未来任何给 backend 加本地卷都会踩经典坑**——命名卷首次挂载是 root 属主,非 root 进程写不进 → PermissionError。防线(加卷时必做):Dockerfile 加 ENTRYPOINT 启动前 `chown` 卷目录,或用固定 UID(`useradd -u 10001`)+ 卷初始化,禁止裸加卷
|
||||
|
||||
### 7. TLS / HTTPS(✅ 已落地 2026-08-09,走②前置 Caddy)
|
||||
|
||||
> **上机确认结论**:此前是裸公网 80 直连 frontend nginx,无任何加密。已按用户选定方案②落地。
|
||||
|
||||
- **已实施(2026-08-09)**:
|
||||
- `docker-compose.prod.yml`:frontend 宿主端口 **`80:80`→`8080:80`**(容器内 nginx 仍监听 80,Caddy 经 compose 网络 `frontend:80` 反代);新增 **caddy** 服务(`caddy:2-alpine`,`80:80`+`443:443`,挂 `Caddyfile` + `caddy_data`/`caddy_config` 卷);volumes 加 caddy_data/caddy_config
|
||||
- **`/root/scilit/Caddyfile`**:`oncolit.gonsun.com { reverse_proxy frontend:80 }`(**必须写 compose 服务名 `frontend:80`**——8080 是宿主映射、compose 网络内服务在容器端口 80;写 8080 会连不上)
|
||||
- **证书自动签发成功**:Let's Encrypt,经 **tls-alpn-01** 挑战(443 可达,说明腾讯云安全组 443 已放行);80 端口 Caddy 自动 **308 跳转 HTTPS**
|
||||
- `PUBLIC_BASE_URL`:`http://`→`https://oncolit.gonsun.com`;`CORS_ORIGINS` 追加 `https://oncolit.gonsun.com`(改 .env 后 backend 需重启生效)
|
||||
- **验证**:`curl -k https://oncolit.gonsun.com` → 200 + 前端 HTML;`/health` 经 Caddy→nginx→backend 返回 `db: ok`;证书 CN=oncolit.gonsun.com,90 天自动续期
|
||||
- **⚠️ frontend 宿主 8080 已收紧(2026-08-09)**:`8080:80` → **`127.0.0.1:8080:80`**(只绑回环,公网无法直连绕过 Caddy;排障时本机 `curl 127.0.0.1:8080` 仍可用;Caddy 经 Docker 网络走 `frontend:80` 不受影响)。安全组仍建议只放行 80/443(禁 3000/2222 对公网),在腾讯云控制台配置
|
||||
- **✅ gitea TLS 已落地(2026-08-09)**:DNS 已加 `gitea.oncolit.gonsun.com` → `123.207.9.209`;Caddyfile gitea 子站块已启用(`gitea.oncolit.gonsun.com { reverse_proxy gitea:3000 }`),证书经 tls-alpn-01 自动签发;gitea `ROOT_URL`/`DOMAIN`/`SSH_DOMAIN` 已改 `https://gitea.oncolit.gonsun.com`。**⚠️ 本地 git remote 需同步改 https**:`http://123.207.9.209:3000/scilit/backend` → `https://gitea.oncolit.gonsun.com/scilit/backend`;SSH clone 地址变为 `gitea.oncolit.gonsun.com:2222`
|
||||
- **⚠️ 服务器本地改动必须回流 gitea(v17 补,2026-08-10)**:TLS/Caddy 全链路改动(compose frontend→8080、caddy 服务+卷、Caddyfile、gitea 域名、`127.0.0.1:8080` 收紧)是 **2026-08-09 直接在服务器上手改的,从未提交到 gitea**(实测:服务器 `docker-compose.prod.yml` 与仓库版 diff 35 行,仓库版还是 `80:80` + `123.207.9.209` + 无 caddy 服务;`Caddyfile` 服务器 8-09 版也不在仓库根)。**后果**:服务器一旦 git-ify checkout 仓库版,这些改动被覆盖 → **HTTPS/Caddy/gitea 域名全丢**。**环节(v17 新增,与 §2"服务器 git pull"闭环)**:①把服务器 `docker-compose.prod.yml`(TLS 版)取回本地覆盖仓库版、`Caddyfile` 加入仓库根 → 提交 `chore: prod compose 同步 TLS/Caddy + gitea 域名` → `git push`(gitea 成为权威,含 TLS);②服务器 git-ify:`yum install git` + `git init -b main` + `remote add origin` + `git fetch && git checkout --force origin/main`(被跟踪文件覆盖为最新,未跟踪的 `.env`/`tmp_*.py` 保留)→ `docker compose config --quiet` 验证 caddy 服务仍在。**此后日常更新 = `git pull`,杜绝手工改与仓库不一致**(对齐 hotfix.sh 的"强制回流"纪律)
|
||||
- **未做**:registry 的 TLS(二期 §2——registry 走明文 HTTP,公网 IP 下 token 有嗅探面,须监听内网或加 TLS)。前端 nginx 容器内直接终止的路线①未采用
|
||||
- **⚠️ 域名前提**:Caddy 自动 HTTPS 要求域名 DNS 已解析到本机(oncolit.gonsun.com → 123.207.9.209 ✓)
|
||||
|
||||
### 8. 日志落盘(v11 提前到一期——日常运维刚需)
|
||||
|
||||
> **原放二期 §4,提前到一期**:查日志是日常运维刚需,镜像优先部署每次重建容器、json-file 日志随容器删除即丢——与北极星"可观测/可恢复"直接相关,不该等二期。一期就能做(前端 nginx 早已挂卷,后端对齐即可);二期只补 Loki/Prometheus 等高级归集。
|
||||
|
||||
- **后端 uvicorn 日志挂宿主机卷**(对齐前端已挂 `/var/log/scilit/nginx`):backend 挂 `/var/log/scilit/backend`,uvicorn 配置 `--access-logfile`/`--error-logfile` 指向该卷内文件(否则默认 stdout 走 json-file、随容器删除即丢);前端 nginx 侧已有 `/var/log/scilit/nginx` 挂载;**⚠️ backend 非 root 写权限(A——v11 的 §8 挂卷正好激活 P8 预告的坑)**:backend 容器是 `USER scilit` 非 root(backend/Dockerfile:29),主机绑定目录 `/var/log/scilit/backend` 是 **root 属主**——scilit 用户写不进去 → **首跑日志落盘即 PermissionError**(nginx:alpine 以 root 跑、无此问题)。**按 §6 P8 现成结论处理,缺一不可**:挂卷前宿主机 `chown <scilit_uid> /var/log/scilit/backend`,或固定 UID(`useradd -u 10001`)+ 卷初始化——**§8 挂卷不配权限处理,一期首跑必崩**
|
||||
- **⚠️ 挂卷后 json-file 轮转失效(容易漏)**:日志挂宿主机卷后,docker 自带 10m×3 轮转**不再覆盖这块日志**——须另配宿主机 **logrotate**(nginx 的 `/var/log/scilit/nginx` 同样要查),否则卷无限涨
|
||||
- **⚠️ 轮转机制二选一(P9)**:宿主机 logrotate **或** uvicorn `RotatingFileHandler` **只能选一种**——两个都配会互相 rename 竞争、日志错乱。**定案:用宿主机 logrotate(下条 copytruncate 细则),uvicorn 保持 stdout 落盘、不配 RotatingFileHandler**
|
||||
- **logrotate 必须 `copytruncate`(v6 执行级真坑)**:容器内进程**不响应日志文件的 rename**——普通 logrotate(rename + create)配了也白配,文件经旧句柄继续涨。**必须配 `copytruncate`,或轮转时发 USR1 让进程重开文件句柄**,否则卷照样无限涨;**⚠️ 配置落地(J)**:在宿主机建 `/etc/logrotate.d/scilit-backend`(root 创建)——一个配置文件同时覆盖 backend + nginx 两段路径:`/var/log/scilit/backend/*.log /var/log/scilit/nginx/*.log { daily; rotate 7; compress; delaycompress; missingok; notifempty; copytruncate }`(**两段都必须 `copytruncate`**,此块已含);配好即由宿主机 `cron.daily` 每日自动轮转,无需重启任何服务
|
||||
|
||||
---
|
||||
|
||||
## 二期:CI + Registry 全自动化 + 可观测
|
||||
|
||||
### 1. Gitea Actions(构建即验证,唯一构建入口)
|
||||
- **当前 CI 从未生效**:`.github/workflows/ci.yml` 是 GitHub 格式,Gitea 不读;需迁到 `.gitea/workflows/`(Gitea Actions 格式)
|
||||
- 迁移现有 ci.yml 逻辑:backend(pgvector:pg16 service + pytest + ruff)+ frontend(vue-tsc + build)→ `.gitea/workflows/ci.yml`
|
||||
- 部署 **act_runner**(服务器容器),注册到 Gitea
|
||||
- **act_runner 需 Docker 能力才能构建镜像(v5 补)**:挂载宿主 `docker.sock`(复用宿主 daemon,简单)或 DinD + privileged(隔离强但重)——落地时确认 runner 内能 `docker build`
|
||||
- **CI 与手动部署共享同一 flock(v6 补)**:Phase 2 CI 触发自动部署时,脚本必须用**同一 `/tmp/scilit-deploy.lock` 路径**——否则 CI 与手动部署仍可能并发抢 tag/versions.log
|
||||
- 启用 Gitea Actions(compose 加 `GITEA__actions__ENABLED: "true"`)
|
||||
|
||||
### 2. Container Registry(生产只 pull,不构建)
|
||||
- Gitea 1.27 原生支持 Container Registry(`<host>:3000/scilit/backend`)
|
||||
- 生产 docker daemon 配 **insecure-registries**(Gitea 无 HTTPS):`["123.207.9.209:3000"]`;**⚠️ 改 daemon.json 后必须重启 docker(P6)**:`systemctl restart docker` 会**重启本机所有容器**(除非预先设 `live-restore: true`)→ 属停机操作,**必须在维护窗口手动做**,不能夹在 deploy.sh 里静默执行(否则一次"配 registry"把整栈全重启一遍)
|
||||
- **⚠️ 公网 IP + HTTP registry 凭证嗅探风险(v7 补)**:`123.207.9.209` 是**公网 IP**,registry 走明文 HTTP——token 在公网/同网段可被嗅探。原「内网可信」假设**不成立**。必须:①registry 仅监听内网/防火墙限定来源,或 ②反代加 TLS(长期);否则 CI push / 生产 pull 的 token 有泄露面
|
||||
- **registry 需认证**:Gitea registry 非匿名,CI push + 生产 pull 都需 `docker login <host>:3000`(token)——token 存服务器 CI secrets / 生产凭据文件,不进 git;**token 有有效期 + 凭据文件本身敏感 → 纳入离线备份(同 `.env` 级)+ 定轮换策略**,过期/泄露即换,写进 docs/10
|
||||
- **registry 盘余量(v6 补)**:registry 复用 gitea_data 卷——§0 扩的是 pgdata 吃紧盘,**二期前必须确认 gitea_data 所在盘余量**,否则 registry 满、镜像推不上去
|
||||
- **二期 image: 改 registry 全地址(v3 提过、v7 固化到二期主体)**:二期 compose 的 image: 必须为 `123.207.9.209:3000/scilit/backend:${BACKEND_TAG:-latest}`(frontend 同理)——**CI push 地址与生产 pull 地址必须完全一致**(同一全限定 registry 前缀),否则 `docker compose pull` 拉的是无 registry 前缀的本地名、推不下来
|
||||
- CI 构建镜像 → 推 registry(打 git-sha 标签)→ 生产 `docker compose pull && up -d`
|
||||
- 保留策略:registry 侧定期清旧版本
|
||||
|
||||
### 3. 生产部署链路(二期形态)
|
||||
- push → CI 构建(带测试)→ 推 registry → 生产 `git pull`(compose 文件)+ `docker compose pull`(镜像)+ `up -d`(migrate 前置 + 健康门控)
|
||||
- **pull 必须限定服务**:`docker compose pull backend worker frontend migrate`——全局 pull 会连带尝试更新 postgres/gitea 等基础设施(尤其 `:latest` 镜像),造成"无意中升级基础设施";**pull 范围与 up 范围一致**
|
||||
- 仍走 deploy.sh 包装(一期脚本),把 build 换成 pull;**迁移同样用 `docker compose run --no-deps --rm backend alembic upgrade head`(K——必加 `--no-deps` 防 migrate 服务重复跑迁移;退出码非 0 即中止,N3 一致性)→ `up -d --no-deps backend worker frontend`**
|
||||
- **健康门控**:healthcheck + depends_on service_healthy 已有;坏版本自动不接流量,换旧 tag 回滚,分钟级恢复
|
||||
- **worker 健康检查形态(N6 落可执行方案——现为弱探活)**:worker 无 HTTP 端点,compose 现有 `grep -q arq /proc/1/cmdline` 只探**进程存在**、不探**消费能力**——坏 worker 会被判健康。**落为可执行(写进 docs/10 + worker 实现)**:①**worker 侧提供心跳键**——ARQ 启动钩子每 N 秒 `SETEX arq_worker_heartbeat 60 <pid>`(复用已有 `REDIS_URL`,无需新 HTTP 端点);②**compose healthcheck 改探心跳——⚠️ 用 python 探,不用 redis-cli(V5,真实会踩)**:worker 镜像是 `python:3.12-slim` 底(backend/Dockerfile,只装 libpq-dev/curl),**不含 redis-cli**——healthcheck 里写 `redis-cli` 会 `not found` → worker **永远 unhealthy** → 部署健康门控反而卡死。**改用容器内已装的 redis-py(ARQ 依赖,零镜像改动)**:`python -c "import redis,sys;r=redis.Redis(host='redis',port=6379,password='${REDIS_PASSWORD}');sys.exit(0 if r.get('arq_worker_heartbeat') else 1)"`;替代方案:worker 镜像 `apt install redis-tools` 装 redis-cli(约几 MB)。探 TTL 内有效 → healthy——探**活性**而非进程存在,卡死/僵死的 worker 心跳过期 → unhealthy → 部署健康门控不再因"进程在"误放行。若后续 worker 挂独立 HTTP 服务,改探 `/healthz` 亦可,心跳键是当前最小改动
|
||||
- **单机停机窗口(写进文档)**:单机无零停机(蓝绿/滚动需多实例);容器重建 + 健康检查 start_period 20s → **部署窗口 ~30s-1min**,选低峰执行;迁移+重建时更久。**"4 workers"语义澄清(v6)**:指 [backend/Dockerfile:34](backend/Dockerfile#L34) `uvicorn --workers 4`——**单个 backend 容器内的 4 个 uvicorn worker**,非 4 个容器;另一个 worker 容器(ARQ)单独存在。停机窗口按单容器重启估算即可;**外部反向代理优雅(E6)**:若前置有反代/LB(见一期 §7),backend 容器重建瞬间 upstream 会短暂 502——反代侧配健康检查剔除 / `proxy_next_upstream`,或接受该次 TCP 闪断(keep-alive 复用连接失败会重连);当前是否有反代需上机确认
|
||||
|
||||
### 4. 可观测增强
|
||||
- 已有:SENTRY_DSN、日志轮转(json-file 10m×3)、healthz + 健康门控
|
||||
- **日志挂卷 + logrotate 已提前到一期 §8(v11)**:后端日志挂卷、logrotate copytruncate、轮转机制二选一(P9)见**一期 §8**——二期不重复,此处只补高级归集
|
||||
- 补:日志归集(如 Loki 或 filebeat → ES)、基础监控(cAdvisor/node-exporter + Prometheus + Grafana)、告警(磁盘/健康/错误率)
|
||||
|
||||
### 5. feature flag(部署/发布解耦)
|
||||
- 代码常上、功能 flag 控制显隐;坏功能一键关,不靠回滚镜像
|
||||
- 用环境变量/配置中心实现,后续按需引入
|
||||
|
||||
---
|
||||
|
||||
## 可选支持层:本地 Docker(不强制)
|
||||
|
||||
- 定位:环境一致是"提前暴露差异"的手段,主要靠 **CI 集成测试**挡差异(用生产同镜像起 Postgres 跑测试),不靠本地复现
|
||||
- 若做(轻量,不迁 38G):WSL2/Docker Desktop 跑 dev compose 中间件,本地原生 uvicorn;价值是 Dockerfile 本地 build 验证 + 冒烟
|
||||
- **不影响一期/二期进度,可随时后补**
|
||||
|
||||
---
|
||||
|
||||
## 关键文件
|
||||
|
||||
| 文件 | 改动 |
|
||||
|---|---|
|
||||
| `docker-compose.prod.yml` | backend/worker/frontend 显式 `image:`;**`deploy.resources.limits` 资源上限(P13)**——backend/worker/es 至少设 memory limit(防单容器 OOM 拖垮宿主,es 已有 `ES_JAVA_OPTS` 但未设 cgroup 上限);gitea 加 Actions/registry 配置 |
|
||||
| `.gitea/workflows/ci.yml`(新) | 由 `.github/workflows/ci.yml` 迁移,Gitea Actions 格式 |
|
||||
| `deploy/*.sh`(新) | deploy / rollback / hotfix / versions.log |
|
||||
| `backend/Dockerfile`、`frontend/Dockerfile.prod` | 构建加速 + 卫生 |
|
||||
| `backend/.dockerignore`、`frontend/.dockerignore` | 构建上下文排除 |
|
||||
| `.gitattributes`(新) | `* text=auto eol=lf` |
|
||||
| **应用 `/health` 端点(v6 重申)** | 健康门控/回滚判定全依赖它——backend 已有 `/health`(nginx 已代理);**新接手者不可漏**,若未来拆分服务需各提供 |
|
||||
| `docs/10-生产部署文档.md` | **既有部署操作手册**,§12 重写为镜像优先流程 + 迁移规范 + 恢复演练(v5 注明:本文件 docs/16 是方案,docs/10 是操作手册,分工不同,不冲突) |
|
||||
| 记忆 `deploy_image_first.md`(新) | 决策 + 脚本用法 |
|
||||
|
||||
---
|
||||
|
||||
## 验证
|
||||
|
||||
**一期**:
|
||||
- 本地 `deploy.sh --dry-run` 只打印命令;Dockerfile 静态核对
|
||||
- 用户服务器首跑:`/health` db ok、前端 200、versions.log 记 sha
|
||||
- 回滚演练:rollback.sh 回上一 sha,无新迁移、服务正常
|
||||
- 恢复演练:pg_restore 到临时库验证数据完整
|
||||
- 首次切镜像会重建容器(migrate 跑迁移),需用户确认窗口
|
||||
|
||||
**二期**:
|
||||
- CI 触发 push → `.gitea/workflows` 跑 lint + pytest + 前端构建,全绿
|
||||
- CI 构建镜像推 registry,生产 `docker pull 123.207.9.209:3000/scilit/backend:<sha>` 成功
|
||||
- 生产 pull + up -d,健康门控生效(坏版本不接流量)
|
||||
- 可观测:Grafana 出图、告警规则触发一次
|
||||
|
||||
---
|
||||
|
||||
## 首次上机确认清单(阻塞级,先查清再动手)
|
||||
|
||||
| # | 待确认 | 出处 | 判定动作 |
|
||||
|---|---|---|---|
|
||||
| 1 | ✅ **backup 落地**(原无 crontab、脚本没跑) | §4 备份 | 已解决:新建 `/root/scilit/scripts/backup.sh`(compose-exec pg_dump)+ crontab `0 3 * * *` + 手动验证成功 |
|
||||
| 2 | ✅ **磁盘扩至 100G**(2026-08-08/09 分区分文件系统补齐) | §0 扩容 | 已完成:`growpart /dev/vda 1` + `xfs_growfs /` → 100G、可用 25G、76% |
|
||||
| 3 | ✅ **gitea/postgres/redis/es/minio 同 compose 文件** | §2 脚本化 | 已实查:全部属 `/root/scilit/docker-compose.prod.yml`(`docker inspect` label 确认);同目录另有 dev 版 `docker-compose.yml`(**命令必须带 `-f docker-compose.prod.yml`**,否则读到 dev 文件报 no such service) |
|
||||
| 4 | compose 是否 v2.x(`service_completed_successfully` 依赖 v2,非 v1) | §3 迁移 | 实查 v2(依赖门控已工作:2026-08-09 migrate 失败即拦下 backend) |
|
||||
| 5 | ✅ **backup.sh 生产连库路径**(postgres `ports: []`,宿主机 localhost 连不上) | §4 备份 | 已解决:统一为 `docker compose exec -T postgres pg_dump`(不经宿主端口) |
|
||||
| 6 | **.dockerignore 脱离 git**(`.gitignore` 第 50 行忽略了它) | §6 卫生 | **已定修复:从 `.gitignore` 移除 `.dockerignore`**(两个 .dockerignore 进 git),执行项 |
|
||||
| 7 | ✅ **compose 服务名统一** | §2 脚本化 | 已实查:prod compose 为 `postgres`,采集命令用对名字 |
|
||||
| 8 | ✅ **TLS 已落地**(走②前置 Caddy) | 一期 §7 TLS | 已完成:frontend→8080、Caddy 80/443、证书签发、PUBLIC_BASE_URL 同步 https;⚠️ 安全组需禁 8080/3000/2222 公网(只放 80/443) |
|
||||
|
||||
---
|
||||
|
||||
## 增强级(后续可选,不阻塞)
|
||||
|
||||
- **同机蓝绿(停机窗口缓解)**:~30s-1min 中断若落在业务高峰不可接受,预留**同机蓝绿**——两份 backend 容器 + nginx upstream 切换(后端日志挂卷 + 镜像版本化已为此铺路);先记下,有需要再做
|
||||
- **hotfix 脆弱窗口**:docker cp 进容器的修复不进镜像,容器一重启即丢——hotfix.sh 末尾强制收口提醒 + 限时正式部署(已落地于 §2)
|
||||
|
||||
---
|
||||
|
||||
## 二次反查新发现(2026-08-08,已验证)
|
||||
|
||||
> 独立反查补充,非吸收外部意见。两条真 bug + 六条设计漏洞。
|
||||
> **v5 更新:** 本节 v3 的 6 条已全部固化到主体现——§1 FRONTEND_TAG/前端 healthcheck/双 tag 原子回滚、§2 工作区检查 + 双 tag log、§3 判定统一为 log、§5 count 制、§6 .dockerignore 定修复。本节保留为历史记录。
|
||||
|
||||
**真 bug:**
|
||||
- **`.dockerignore` 被 `.gitignore` 忽略**(根 `.gitignore` 第 50 行):backend/frontend 两个 `.dockerignore` 未被 git 跟踪(`git ls-files` 确认),但 §6 要把它们作为部署单元随 git 分发——服务器 pull 不到。**修复:从 `.gitignore` 移除 `.dockerignore`**,或明确它为服务器本地手工同步
|
||||
- **backup.sh 生产连库路径不成立**:仓库版默认 `PG_HOST=localhost:5432`,但 postgres `ports: []` 不发布端口,宿主机 `pg_dump` 连接拒绝。**统一到仓库版前先定连库机制**(`docker compose exec postgres pg_dump` 或加 `127.0.0.1:5432:5432` 映射),已入上机清单 #5
|
||||
|
||||
**设计漏洞:**
|
||||
- **frontend 缺 healthcheck**:只有 `depends_on: backend: service_started`(容器起来即可),无自身健康门控——"坏版本不接流量"对前端失效,补 `curl -sf http://localhost/` 或 nginx 配置校验
|
||||
- **frontend tag 插值缺失**:仅 `BACKEND_TAG`,frontend 是独立 nginx 镜像,需 `FRONTEND_TAG`;**回滚必须 backend+frontend 双 tag 原子切换**,versions.log 记录两个
|
||||
- **镜像治理策略打架**:§1 "保留 N=5-10 tag"(count 制)vs §5 `prune until=168h`(age 制)——低频部署时 7 天外的保留 tag 被 age-prune 清掉 → 回滚点丢失。**统一:应用镜像只按 count 清理,age-prune 只作用于 dangling/build cache**
|
||||
- **deploy.sh 未查工作区干净**:生产机 `git pull` 前需 `git status --porcelain` 为空,否则 hotfix 残留导致 pull 冲突;冲突即中止
|
||||
- **破坏性判定扫描时机**:回滚时扫当前工作区 ≠ 回滚目标版本——destructive 文件可能已在后续版本删除/改名。**改为 deploy 时基于 `git diff <prev>..<sha> -- alembic/versions/` 检测 drop/alter 并写进 versions.log,回滚读 log**
|
||||
- **一期/二期 image: 写法切换**:本地 build 用 `scilit/backend:<sha>`,二期 pull 用 `123.207.9.209:3000/scilit/backend:<sha>`——切换时 compose 的 image: 字段必须改为全限定 registry 地址,需在文档标注
|
||||
|
||||
---
|
||||
|
||||
## 风险与注意
|
||||
|
||||
- **磁盘**:✅ 已扩至 100G(2026-08-08);绝不 `docker builder prune -a`(毁缓存)、绝不 `docker compose down -v`(毁数据卷)
|
||||
- **restart 策略(P7 已查实存在,无需新增)**:prod compose 全部服务(postgres/redis/es/minio/backend/worker/frontend/gitea)已是 `restart: unless-stopped`——**服务器重启后栈自动拉起**;migrate `restart: "no"`(一次性服务,正确)。冷启动/回滚场景都依赖此自愈,部署后 `docker compose ps` 复核各容器 restart 策略未被误改
|
||||
- **TZ 时区决策(E7)**:项目刻意用 **UTC 存储**(ARQ 任务时间全 UTC,见 CLAUDE.md;DB 列均为 TIMESTAMPTZ/UTC)——**不设 `TZ=Asia/Shanghai`**,避免容器本地时间与 DB UTC 混读;日志/时间戳用 ISO8601 带时区(如 `+08:00`),前端展示层本地化。若日后运维强烈偏好本地时区可统一设 TZ,但须知 DB 仍 UTC、两时区并存易混——先记录决策,不擅自改
|
||||
- **insecure-registries**:Gitea 无 HTTPS,生产/runner 需配明文 registry——**公网 IP 下「内网可信」不成立(v7)**:registry 仅监听内网/防火墙限定来源,或反代加 TLS,否则 CI/生产 token 有泄露面(见二期 §2)
|
||||
- **CI 一次性配置门槛高**:act_runner 注册、registry 开启、Gitea Actions 启用——过渡期注意测试
|
||||
- 生产部署由用户执行,agent 不直接 SSH;部署前确认;计划批准≠执行绿灯
|
||||
@@ -0,0 +1,158 @@
|
||||
# 17. 生产数据迁移实施方案
|
||||
|
||||
> **日期:** 2026-08-09 **版本:** v1.0
|
||||
> **关联:** [16-部署运维方案.md](16-部署运维方案.md)(deploy.sh 部署流程)、[10-生产部署文档.md](10-生产部署文档.md)(操作手册)
|
||||
> **状态:** 待执行——生产 DB 落后本地 24 个迁移,本方案拆分 4 批,大数据量批次延后手工执行
|
||||
|
||||
---
|
||||
|
||||
## 1. 背景与现状
|
||||
|
||||
生产服务器(123.207.9.209)的数据库 schema 停留在 **2026-07-17**(alembic head = `6b662a8c5235`),而本地开发代码已演进到 **2026-07-30**(head = `55105f0bb1d7`),**中间有 24 个迁移待执行**。
|
||||
|
||||
关键数据规模(影响迁移耗时与锁表风险):
|
||||
- `global_literature`:**123 万行**
|
||||
- `global_literature_tags`:**348 万行**
|
||||
- `global_tags`:2.1 万行
|
||||
|
||||
**⚠️ 核心约束:** 24 个迁移形成**线性链**(01→24 顺序依赖),无法跳过任何一个直接跑到 head。所以"分开做"= 把链切成 4 段,每段执行到中间 checkpoint,**大数据量/锁表操作集中在后段**,由人工控制执行时机。
|
||||
|
||||
---
|
||||
|
||||
## 2. 迁移链全景(24 个迁移,生产 head → 本地 head)
|
||||
|
||||
| # | Revision | 内容 | 数据量/风险 |
|
||||
|---|---|---|---|
|
||||
| 01 | `cb07d6b1df01` | 19 列 JSON→JSONB(全表重写)+ 3 新列 + entry_terms + search_tsv 触发器重建 + 8 索引(B-tree×4 + GIN×4) | 🔴 **重:全表重写** |
|
||||
| 02 | `6492b887b779` | `meshed_date` 列 + UPDATE 回填(`date_completed` 非空行) | 🟠 中:全表回填 |
|
||||
| 03 | `1421ea169bb6` | 5 个日期字段 TIMESTAMPTZ→DATE | 🔴 **重:锁表** |
|
||||
| 04 | `f3b135d62407` | pipeline_runs.processed_date | 🟢 轻 |
|
||||
| 05 | `0ee585329fc6` | pipeline_runs.metadata | 🟢 轻 |
|
||||
| 06 | `db1cc822f2da` | global_journals.nlm_subsets | 🟢 轻 |
|
||||
| 07 | `34bc08516f3a` | global_literature.is_preprint | 🟢 轻 |
|
||||
| 08 | `d641e2f7a4ee` | auid_data / cois_statement / vernacular 相关 | 🟢 轻 |
|
||||
| 09 | `52ec204acfd9` | pharmacological_actions JSONB | 🟢 轻 |
|
||||
| 10 | `38bb4e1f8498` | investigators / personal_name | 🟢 轻 |
|
||||
| 11 | `cac545862583` | 新建 user_saved_filters 表 | 🟢 轻 |
|
||||
| 12 | `d8f6c3563587` | 筛选列 B-tree×4 + mesh_headings GIN | 🟠 中:大表建索引 |
|
||||
| 13 | `a4b7c8d9e0f1` | journal_iso trgm GIN | 🟠 中:大表建索引 |
|
||||
| 14 | `e5f6a7b8c9d0` | search_tsv 触发器函数(作者 simple 词典) | 🟢 轻 |
|
||||
| 15 | `d9e7c8b1a2f3` | authors trgm GIN | 🟠 中:大表建索引 |
|
||||
| 16 | `e0f1a2b3c4d5` | search_tsv 触发器重建(+chemical/gene,无回填) | 🟢 轻 |
|
||||
| 17 | `f1a2b3c4d5e6` | author_names_text 列 + UPDATE 回填 + trgm 索引 | 🔴 **重:全表回填** |
|
||||
| 18 | `g0h1i2j3k4l5` | search_tsv 全量回填(+mesh/keywords)+ 触发器 | 🔴 **重:全表回填** |
|
||||
| 19 | `e0764f6d7c21` | tag_ids ARRAY + BRIN×2 + 部分索引×4 + tag_ids GIN | 🟠 中:大表建索引 |
|
||||
| 20 | `b01b8f27c596` | **tag_ids 回填**(348 万行 array_agg 聚合) | 🔴 **重:大数据量回填** |
|
||||
| 21 | `af4a8b2ec873` | search_tsv 全量回填(修复 mesh_headings key)+ 触发器 | 🔴 **重:全表回填** |
|
||||
| 22 | `d166cde6083b` | volume/issue VARCHAR(50)→VARCHAR(200) | 🟢 轻 |
|
||||
| 23 | `95c18ebf31e4` | volume/issue VARCHAR(200)→Text | 🔴 **重:锁表** |
|
||||
| 24 | `55105f0bb1d7` | journal_iso/pages VARCHAR(100)→Text | 🔴 **重:锁表** |
|
||||
|
||||
---
|
||||
|
||||
## 3. 分段原则
|
||||
|
||||
1. **链是线性的**(01→24),不能跳段;分段 = 每批执行到中间 checkpoint,不改变迁移内部顺序。
|
||||
2. **大数据量放后边**:🔴 重活集中在批次 1(链头,无法避免,见下)和**批次 4**(全部延后)。
|
||||
3. **新代码依赖前置 schema**:批次 1 的 JSON→JSONB / 日期类型变更,本地模型代码已经按 JSONB/Date 定义——**在批次 1 完成前,新代码无法运行**。这是必须最先执行、无法延后的部分。
|
||||
4. **批次 4 = 最重**:含 348 万行 tag_ids 回填 + 3 次全量 search_tsv 回填 + 2 次锁表类型变更,**全部延后到深夜低峰手工跑**。
|
||||
|
||||
---
|
||||
|
||||
## 4. 批次划分与目标 Checkpoint
|
||||
|
||||
| 批次 | 覆盖迁移 | 目标 checkpoint | 脚本参数 | 内容摘要 | 建议时段 |
|
||||
|---|---|---|---|---|---|
|
||||
| 1 | [01]–[03] | `1421ea169bb6` | `./migrate_prod.sh 1` | JSON→JSONB 全表重写 + meshed_date 回填 + 日期锁表 | ⚠️ **必须与发新代码同窗口**(含 destructive,见下) |
|
||||
| 2 | [04]–[11] | `cac545862583` | `./migrate_prod.sh 2` | 纯加列/新表,additive | 可提前,任意时段 |
|
||||
| 3 | [12]–[16] | `e0f1a2b3c4d5` | `./migrate_prod.sh 3` | 索引(大表建)+ 触发器重建,additive | 可提前,建议低峰 |
|
||||
| 4 | [17]–[24] | `head`(`55105f0bb1d7`) | `./migrate_prod.sh 4` | **大数据量回填 + 类型锁表(含 destructive)** | **新代码已部署后,深夜低峰** |
|
||||
|
||||
> **⚠️ destructive 窗口纪律(2026-08-10 修正,防止批次 1 单独跑崩老代码):**
|
||||
> - **批次 1 含 `1421ea169bb6`(5 日期字段 TIMESTAMPTZ→DATE)——类型收窄,属 destructive**。老代码把 `date_completed`/`pubmed_revised` 等读成 datetime,迁移后 DB 返回 `date`,`datetime` 专属调用会崩。**因此批次 1 绝不能脱离代码更新单独跑**——必须和 build 新镜像 + 发新代码同一维护窗口(迁移完几秒内新镜像接管),或新代码先发再跑批次 1(但新代码依赖 JSONB,见 §3 约束,建议同窗口)。
|
||||
> - **批次 2/3 纯 additive**(加列/新表/索引/触发器重建),老代码完全兼容,**可在部署前任意时段提前跑**,缩小维护窗口。
|
||||
> - **批次 4 含 `95c18ebf31e4`/`55105f0bb1d7`(VARCHAR→Text)**——类型加宽、老代码兼容,但按 destructive 判定仍算;**必须在新代码已部署后、深夜低峰跑**(大数据量回填 + 锁表)。
|
||||
> - 推荐执行序:**批次 2/3(提前)→ 维护窗口:build 新镜像 → 批次 1 → 发新 backend/worker/frontend → 深夜:批次 4**。批次 1 与发代码之间的窗口必须控制在分钟级内。
|
||||
|
||||
---
|
||||
|
||||
## 5. 前置条件(执行迁移前必须完成)
|
||||
|
||||
以下由部署流程(docs/16 §2 deploy.sh)或人工准备:
|
||||
|
||||
1. **新 backend 镜像已构建**(必须包含 24 个新迁移文件)。生产镜像当前是 07-17 旧版,**用旧镜像跑迁移会报 `Can't locate revision '6b662a8c5235'`**(2026-08-09 事故根因)。验证方式:
|
||||
```bash
|
||||
docker compose -f docker-compose.prod.yml build backend
|
||||
```
|
||||
> **构建加速已落地(2026-08-10):** backend Dockerfile 已加腾讯 pip/apt 源 + BuildKit 缓存,frontend Dockerfile.prod 已加腾讯 npm 源 + `npm ci`。**首次 build 约几分钟**(pip 全量从腾讯源装),**之后代码级 build 秒~1 分钟**(只 `COPY . .`)。构建前提:宿主机 daemon 已配腾讯 registry-mirrors(✅ 已配)、BuildKit 需 Docker 22.06+/23.05+(✅ 29.6.1)。
|
||||
2. **⚠️ 批次 1 必须与发新代码同窗口**(见 §4 destructive 纪律):批次 1 的日期收窄迁移会让老代码读崩,**不能提前单独跑**。批次 2/3 可提前 additive 跑。
|
||||
2. **数据库快照**(安全网,回滚用):部署前拍全量 `pg_dump -Fc`。
|
||||
3. **postgres 容器 healthy**。
|
||||
4. 脚本位于 `/root/scilit/`(与 `docker-compose.prod.yml` 同目录)。
|
||||
|
||||
---
|
||||
|
||||
## 6. 实施步骤(生产服务器手工执行)
|
||||
|
||||
### 6.1 脚本位置与用法
|
||||
|
||||
脚本:`backend/scripts/migrate_prod.sh`(已随代码提交,部署时同步到 `/root/scilit/`)
|
||||
|
||||
```bash
|
||||
cd /root/scilit
|
||||
chmod +x migrate_prod.sh # 首次
|
||||
./migrate_prod.sh 1 # 批次 1
|
||||
./migrate_prod.sh 2 # 批次 2
|
||||
./migrate_prod.sh 3 # 批次 3
|
||||
./migrate_prod.sh 4 # 批次 4(最重,深夜)
|
||||
```
|
||||
|
||||
脚本内置安全网:
|
||||
- **镜像新鲜度检查**:执行前确认 backend 镜像含本地 head(`55105f0bb1d7`),否则报错并给出修复指引(防 08-09 事故重演)。
|
||||
- **起点校验**:检查 DB 当前 revision 是否等于该批次的预期起点,防止乱序/重复。
|
||||
- **幂等**:DB 已在目标 checkpoint 时直接跳过。
|
||||
- **退出码门控**:迁移命令失败(非 0)即中止,`set -euo pipefail`。
|
||||
|
||||
### 6.2 每个批次的验证点
|
||||
|
||||
```bash
|
||||
# 执行后确认当前版本
|
||||
docker compose -f docker-compose.prod.yml run --no-deps --rm backend alembic current
|
||||
```
|
||||
|
||||
预期结果(逐批):
|
||||
|
||||
| 批次 | 执行后 `alembic current` |
|
||||
|---|---|
|
||||
| 1 | `1421ea169bb6` |
|
||||
| 2 | `cac545862583` |
|
||||
| 3 | `e0f1a2b3c4d5` |
|
||||
| 4 | `55105f0bb1d7 (head)` |
|
||||
|
||||
---
|
||||
|
||||
## 7. 注意事项
|
||||
|
||||
1. **执行序(destructive 窗口纪律)**:**批次 2/3(additive)可提前跑 → 维护窗口内:build → 批次 1 → 发新代码 → 深夜:批次 4**。批次 1(日期收窄)与发代码必须同窗口(分钟级内衔接),批次 4(大数据量回填 + Text 加宽)须在新代码已部署后深夜跑。
|
||||
2. **锁表窗口**:批次 1(日期锁表)、批次 4(volume/issue/pages→Text 锁表)期间,对 `global_literature` 的写入被阻塞。建议选业务低峰。123 万行上的回填(02/17/18/20/21)预计每步数秒到数分钟,总计约 10-20 分钟。
|
||||
3. **批次 4 完成后,DB 即升级到本地最新 head**——代码与 schema 完全对齐。批次 4 未跑时新代码可运行,但 `tag_ids` 为空、搜索 tsvector 覆盖不全(回填数据缺失),功能受限但不报错。
|
||||
4. **失败处理**:若某批次迁移失败,脚本退出非 0;DB 停在失败前一个 revision,可重跑该批次(alembic 只应用未执行的迁移)。涉及数据回退时用 §5 快照(`pg_restore --clean --if-exists`,见 docs/16 §2 rollback.sh)。
|
||||
|
||||
---
|
||||
|
||||
## 8. 回滚预案
|
||||
|
||||
- **迁移失败 / 需回退数据**:使用部署前快照 `pg_restore --clean --if-exists --no-owner -d scilit <快照>` 恢复(docs/16 §2 rollback.sh 完整流程)。
|
||||
- **代码回滚**:换回旧镜像 tag + `up -d --no-deps`(docs/16 §1 双 tag 原子回滚)。
|
||||
- **迁移本身是单向的**:alembic 各迁移均有 `downgrade()`,但**破坏性迁移(类型变更/回填)downgrade 不恢复数据**——正式回滚走快照,不走 downgrade。
|
||||
|
||||
---
|
||||
|
||||
## 9. 执行记录
|
||||
|
||||
| 日期 | 批次 | 执行结果 | 备注 |
|
||||
|---|---|---|---|
|
||||
| (待填) | 1 | | |
|
||||
| (待填) | 2 | | |
|
||||
| (待填) | 3 | | |
|
||||
| (待填) | 4 | | |
|
||||
@@ -0,0 +1,7 @@
|
||||
node_modules
|
||||
dist
|
||||
.git
|
||||
.gitignore
|
||||
README.md
|
||||
*.local
|
||||
.env
|
||||
@@ -1,7 +1,11 @@
|
||||
# 构建加速(2026-08-10):腾讯 npm 源 + npm ci(依赖 lockfile 精确安装)
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM node:20-alpine AS builder
|
||||
WORKDIR /app
|
||||
COPY package.json .
|
||||
RUN npm install
|
||||
# 腾讯 npm 镜像(内网可达,快)
|
||||
RUN npm config set registry https://mirrors.cloud.tencent.com/npm/
|
||||
COPY package.json package-lock.json ./
|
||||
RUN --mount=type=cache,target=/root/.npm npm ci
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
|
||||
|
||||
Vendored
+1
@@ -15,6 +15,7 @@ declare module 'vue' {
|
||||
NModal: typeof import('naive-ui')['NModal']
|
||||
NoteEditModal: typeof import('./components/notes/NoteEditModal.vue')['default']
|
||||
NotificationBell: typeof import('./components/common/NotificationBell.vue')['default']
|
||||
NResult: typeof import('naive-ui')['NResult']
|
||||
PageSkeleton: typeof import('./components/common/PageSkeleton.vue')['default']
|
||||
RouterLink: typeof import('vue-router')['RouterLink']
|
||||
RouterView: typeof import('vue-router')['RouterView']
|
||||
|
||||
@@ -21,8 +21,13 @@ export function usePagination(opts: UsePaginationOptions) {
|
||||
const hasMore = computed(() => page.value < totalPages.value)
|
||||
|
||||
async function goToPage(n: number) {
|
||||
const prev = page.value
|
||||
page.value = n
|
||||
await fetchFn(n)
|
||||
try {
|
||||
await fetchFn(n)
|
||||
} catch {
|
||||
page.value = prev
|
||||
}
|
||||
}
|
||||
|
||||
return { page, pageSize, total, totalPages, hasMore, goToPage }
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, computed, onMounted, onUnmounted, watch } from 'vue'
|
||||
import { useRouter, useRoute } from 'vue-router'
|
||||
import { useRouter, useRoute, onBeforeRouteUpdate } from 'vue-router'
|
||||
import { useAuthStore } from '../../stores/auth'
|
||||
import { NInput, NButton, NEmpty, NSelect, NCheckboxGroup, NCheckbox, NPagination, NIcon, NRadioGroup, NRadio, NSlider, NModal, NPopover } from 'naive-ui'
|
||||
import { SearchOutline, FilterOutline, EyeOffOutline, SettingsOutline } from '@vicons/ionicons5'
|
||||
@@ -28,6 +28,7 @@ const sort = ref('date')
|
||||
const results = ref<LiteratureItem[]>([])
|
||||
const loading = ref(false)
|
||||
const searched = ref(false)
|
||||
const searchError = ref('')
|
||||
const savedPmids = ref<Set<number>>(new Set())
|
||||
|
||||
// ── 筛选参数 ──
|
||||
@@ -62,6 +63,7 @@ const expandedGroups = ref<Record<string, boolean>>({})
|
||||
const showFilters = ref(localStorage.getItem('search:showFilters') !== 'false')
|
||||
const yearCounts = ref<{ year: number; count: number }[]>([])
|
||||
const showCustomYear = ref(false) // 自定义年份范围展开
|
||||
const _skipRouteSync = ref(0) // R31: guard against re-entry from syncSearchToUrl
|
||||
const pageSize = ref(Number(localStorage.getItem('search:pageSize')) || 20)
|
||||
// 从 URL 恢复的页码(syncSearchToUrl 写入)
|
||||
const restoredPage = ref(1)
|
||||
@@ -92,6 +94,7 @@ function onYearSliderChange(val: any) {
|
||||
yearFromStr.value = String(val[0])
|
||||
yearToStr.value = String(val[1])
|
||||
datePreset.value = null
|
||||
showCustomYear.value = false // R31: 滑动时收起自定义年份
|
||||
urlDateFrom.value = ''; urlDateTo.value = '' // P6: clear stale URL dates
|
||||
if (_sliderTimer) clearTimeout(_sliderTimer)
|
||||
_sliderTimer = setTimeout(() => goToPage(1), 250)
|
||||
@@ -273,6 +276,8 @@ const { page, total, goToPage } = usePagination({
|
||||
const gen = ++searchGeneration.value
|
||||
loading.value = true
|
||||
searched.value = true
|
||||
searchError.value = ''
|
||||
total.value = 0
|
||||
try {
|
||||
if (p === 1 && query.value.trim()) {
|
||||
trackAction('search', 'search', query.value.trim(), { sort: sort.value })
|
||||
@@ -361,12 +366,21 @@ const { page, total, goToPage } = usePagination({
|
||||
delete keysetCursors.value[p + 1]
|
||||
}
|
||||
yearCounts.value = data.year_counts || []
|
||||
searchError.value = ''
|
||||
syncSearchToUrl() // 成功时同步 URL
|
||||
} catch (e: any) {
|
||||
if (e?.name === 'CanceledError' || e?.code === 'ERR_CANCELED') return
|
||||
results.value = []
|
||||
searchError.value = e?.response?.status === 401
|
||||
? '请登录后使用搜索功能'
|
||||
: e?.response?.status === 400
|
||||
? e?.response?.data?.detail || '搜索参数有误,请调整后重试'
|
||||
: e?.response?.status === 429
|
||||
? '请求过于频繁,请稍后重试'
|
||||
: '搜索失败,请检查网络或稍后重试'
|
||||
toast.apiError(e, '搜索失败,请重试')
|
||||
}
|
||||
finally {
|
||||
syncSearchToUrl() // P6: sync URL even on error (avoid URL/state desync)
|
||||
if (gen === searchGeneration.value) loading.value = false
|
||||
}
|
||||
},
|
||||
@@ -382,8 +396,13 @@ function restoreFromQuery() {
|
||||
if (route.query.sort && VALID_SORTS.has(String(route.query.sort))) sort.value = String(route.query.sort)
|
||||
if (route.query.date_preset && ['1y','5y','10y','custom'].includes(String(route.query.date_preset))) {
|
||||
datePreset.value = String(route.query.date_preset)
|
||||
yearFromStr.value = ''
|
||||
yearToStr.value = ''
|
||||
if (datePreset.value === 'custom') {
|
||||
if (route.query.year_from) yearFromStr.value = String(route.query.year_from)
|
||||
if (route.query.year_to) yearToStr.value = String(route.query.year_to)
|
||||
} else {
|
||||
yearFromStr.value = ''
|
||||
yearToStr.value = ''
|
||||
}
|
||||
} else if (route.query.date_from || route.query.date_to) {
|
||||
datePreset.value = null
|
||||
if (route.query.date_from) {
|
||||
@@ -486,6 +505,13 @@ onMounted(async () => {
|
||||
await goToPage(restoredPage.value)
|
||||
})
|
||||
|
||||
// R31: 浏览器后退/前进时恢复搜索状态
|
||||
onBeforeRouteUpdate(() => {
|
||||
if (_skipRouteSync.value > 0) return // syncSearchToUrl 触发的路由变化,跳过
|
||||
restoreFromQuery()
|
||||
goToPage(restoredPage.value)
|
||||
})
|
||||
|
||||
/** 同步当前搜索参数到 URL query */
|
||||
function syncSearchToUrl() {
|
||||
const q: Record<string, string> = {}
|
||||
@@ -531,7 +557,9 @@ function syncSearchToUrl() {
|
||||
// 页码持久化到 URL(Keyset 排序使用 keyset 页码,offset 排序使用 offset 页码)
|
||||
const currentPage = KEYSET_SORTS.has(sort.value) ? keysetPage.value : page.value
|
||||
if (currentPage > 1) q.p = String(currentPage)
|
||||
router.replace({ query: q }).catch(() => {})
|
||||
// R24: use push instead of replace to preserve browser back-button history
|
||||
_skipRouteSync.value++
|
||||
router.push({ query: q }).catch(() => {}).finally(() => _skipRouteSync.value--)
|
||||
}
|
||||
|
||||
function resetAllFilters() {
|
||||
@@ -560,6 +588,7 @@ function resetAllFilters() {
|
||||
booleanOp.value = 'and'
|
||||
exactPhrase.value = false
|
||||
showCustomYear.value = false
|
||||
pageSize.value = Number(localStorage.getItem('search:pageSize')) || 20 // R31: 重置每页条数
|
||||
goToPage(1)
|
||||
}
|
||||
|
||||
@@ -897,7 +926,8 @@ const specialTags = computed(() => filterOptions.value?.special_tags || {})
|
||||
</div>
|
||||
|
||||
<PageSkeleton :loading="loading && !results.length">
|
||||
<NEmpty v-if="searched&&!loading&&!results.length" description="未找到匹配文献,尝试修改搜索条件" />
|
||||
<NResult v-if="searchError" status="error" :title="searchError" description="可稍后重试或联系管理员" />
|
||||
<NEmpty v-else-if="searched&&!loading&&!results.length" description="未找到匹配文献,尝试修改搜索条件" />
|
||||
|
||||
<LiteratureCard
|
||||
v-for="item in results"
|
||||
@@ -947,7 +977,7 @@ const specialTags = computed(() => filterOptions.value?.special_tags || {})
|
||||
<template #footer>
|
||||
<div style="display:flex;justify-content:space-between;align-items:center">
|
||||
<span style="font-size:13px;color:var(--text-muted)">已选择 {{ pubTypes.length }} 项</span>
|
||||
<NButton size="small" type="primary" @click="showPubTypeModal = false; goToPage(1)">确定</NButton>
|
||||
<NButton size="small" type="primary" @click="showPubTypeModal = false">确定</NButton>
|
||||
</div>
|
||||
</template>
|
||||
</NModal>
|
||||
@@ -967,7 +997,7 @@ const specialTags = computed(() => filterOptions.value?.special_tags || {})
|
||||
</div>
|
||||
<template #footer>
|
||||
<div style="display:flex;justify-content:flex-end">
|
||||
<NButton size="small" type="primary" @click="showLangModal = false; goToPage(1)">确定</NButton>
|
||||
<NButton size="small" type="primary" @click="showLangModal = false">确定</NButton>
|
||||
</div>
|
||||
</template>
|
||||
</NModal>
|
||||
@@ -983,7 +1013,7 @@ const specialTags = computed(() => filterOptions.value?.special_tags || {})
|
||||
</div>
|
||||
<template #footer>
|
||||
<div style="display:flex;justify-content:flex-end">
|
||||
<NButton size="small" type="primary" @click="showAgeModal = false; goToPage(1)">确定</NButton>
|
||||
<NButton size="small" type="primary" @click="showAgeModal = false">确定</NButton>
|
||||
</div>
|
||||
</template>
|
||||
</NModal>
|
||||
|
||||
@@ -85,9 +85,9 @@ const translated = computed(() => {
|
||||
items.push({ field: `[${tag}] ${label}`, value: `"${val}"` })
|
||||
}
|
||||
}
|
||||
// Date range: YYYY:YYYY[DP] or YYYY/MM/DD:YYYY/MM/DD[DP]
|
||||
// 对 displayText(已展开 #N)扫描,确保历史引用中的 DP 也能被翻译
|
||||
const yrRe = /(\d{4}(?:\/\d{2}\/\d{2})?)\s*:\s*(\d{4}(?:\/\d{2}\/\d{2})?)\s*\[DP\]/g
|
||||
// Date range: supports YYYY:YYYY[DP], YYYY-MM-DD:YYYY-MM-DD[DP], YYYY/MM/DD:YYYY/MM/DD[DP],
|
||||
// YYYY-M-D:YYYY-M-D[DP] (single digit), YYYY-MM:YYYY-MM[DP] (month-only)
|
||||
const yrRe = /(\d{4}(?:[-\/]\d{1,2}(?:[-\/]\d{1,2})?)?)\s*:\s*(\d{4}(?:[-\/]\d{1,2}(?:[-\/]\d{1,2})?)?)\s*\[DP\]/g
|
||||
while ((m = yrRe.exec(displayText)) !== null) {
|
||||
const key = `dp_range_${m.index}`
|
||||
if (!seen.has(key)) {
|
||||
@@ -155,6 +155,8 @@ function addToQuery() {
|
||||
let val = builderValue.value.trim()
|
||||
// 去除用户输入的多余引号
|
||||
val = val.replace(/^['""“]+|['""”]+$/g, '')
|
||||
// R31: 去除内部双引号防止构建非法 PubMed 语法(保留单引号如 don't)
|
||||
val = val.replace(/["""“”]/g, '')
|
||||
if (!val) {
|
||||
message.warning('请输入搜索词')
|
||||
return
|
||||
@@ -242,7 +244,6 @@ function resolveQuery(q: string): string {
|
||||
if (num === undefined) return m // 在引号内,不做替换
|
||||
const found = all.find(e => e.id === `#${num}`)
|
||||
if (!found) {
|
||||
message.warning(`查询编号 ${m} 在历史中不存在,已保留原样`)
|
||||
return m
|
||||
}
|
||||
return `(${found.expanded_query})`
|
||||
|
||||
Reference in New Issue
Block a user