34047007@qq.com
3c85ded216
- Dockerfile 腾讯源 + BuildKit 缓存（backend pip / frontend npm ci），gitea 锁 1.27.1 - .dockerignore 纳入版本管理（防密钥进镜像） - 新增 migrate_prod.sh：24 个迁移分 4 批执行，含镜像新鲜度 + DB 起点校验 - cb07d6b1df01 移除 search_tsv 回填（延迟到 g0h1i2j3k4l5 统一全量回填） - 模型 journal_iso/volume/issue/pages → Text（对应迁移 95c18ebf31e4 / 55105f0bb1d7） - 搜索优化：tsvector 主路径 + COUNT 截断(10000) + pub_date 排序，admin 聚合单查询 - statement_timeout 30s→60s
236 lines
16 KiB
Python
236 lines
16 KiB
Python
"""全局文献库模型：global_literature, global_tags, global_literature_tags, global_journals"""
|
||
|
||
import uuid
|
||
from datetime import date, datetime
|
||
|
||
from sqlalchemy import Boolean, Date, DateTime, ForeignKey, Index, Integer, String, Text, Uuid, UniqueConstraint, func, text
|
||
from sqlalchemy.dialects.postgresql import ARRAY, JSONB, TSVECTOR
|
||
from sqlalchemy.orm import Mapped, mapped_column
|
||
|
||
from app.db import Base, new_uuid
|
||
|
||
|
||
class GlobalLiterature(Base):
|
||
__tablename__ = "global_literature"
|
||
|
||
id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=new_uuid)
|
||
pmid: Mapped[int] = mapped_column(Integer, unique=True, nullable=False)
|
||
title: Mapped[str] = mapped_column(Text, nullable=False)
|
||
abstract: Mapped[str | None] = mapped_column(Text)
|
||
authors: Mapped[dict] = mapped_column(JSONB, default=list)
|
||
doi: Mapped[str | None] = mapped_column(String(500))
|
||
journal: Mapped[str | None] = mapped_column(String(500))
|
||
journal_issn: Mapped[str | None] = mapped_column(String(20))
|
||
journal_iso: Mapped[str | None] = mapped_column(Text) # NLM ISO 缩写 (e.g. "N Engl J Med")
|
||
volume: Mapped[str | None] = mapped_column(Text)
|
||
issue: Mapped[str | None] = mapped_column(Text)
|
||
pages: Mapped[str | None] = mapped_column(Text)
|
||
pub_date: Mapped[date | None] = mapped_column(Date) # 纸质出版日期（期刊卷期日期，纯电子刊则为电子日期）【PubMed: PubDate】
|
||
print_date: Mapped[date | None] = mapped_column(Date) # 纸质出版日期（仅纸质版见刊日期）【PubMed: PPDAT】
|
||
pub_year: Mapped[int | None] = mapped_column(Integer)
|
||
pub_types: Mapped[dict] = mapped_column(JSONB, default=list)
|
||
mesh_headings: Mapped[dict] = mapped_column(JSONB, default=list)
|
||
language: Mapped[str] = mapped_column(String(10), default="en")
|
||
raw_xml_hash: Mapped[str | None] = mapped_column(String(64))
|
||
|
||
# ═══ 新增字段 ═══
|
||
pmc_id: Mapped[str | None] = mapped_column(String(20)) # PMC1234567
|
||
is_oa: Mapped[bool] = mapped_column(default=False) # 开放存取
|
||
keywords: Mapped[dict] = mapped_column(JSONB, default=list) # 作者关键词
|
||
pubmed_revised: Mapped[date | None] = mapped_column(Date) # PubMed 修订日期（纯日期，NLM 仅提供 Y/M/D）
|
||
grants: Mapped[dict] = mapped_column(JSONB, default=list) # 基金资助
|
||
cited_by_count: Mapped[int] = mapped_column(Integer, default=0) # 引用次数
|
||
full_text_sections: Mapped[dict | None] = mapped_column(JSONB) # PMC OA 全文结构化段落（旧，不断新增存 COS）
|
||
full_text_path: Mapped[str | None] = mapped_column(String(255)) # COS 对象键，全文 JSON 新数据存这里
|
||
trial_reg: Mapped[dict | None] = mapped_column(JSONB) # 临床试验注册号 {nct, eudract, chictr}
|
||
study_design: Mapped[dict | None] = mapped_column(JSONB) # 研究设计分类 {primary, sub, design}
|
||
pico: Mapped[dict | None] = mapped_column(JSONB) # PICO {population, intervention, comparator, outcome, sample_size, effect_size}
|
||
citation_status: Mapped[str | None] = mapped_column(String(20)) # MedlineCitation Status: publisher/in-process/medline/...
|
||
date_completed: Mapped[date | None] = mapped_column(Date) # NLM 整条编目记录完成日（含 MeSH 标引在内的全部处理完成）
|
||
meshed_date: Mapped[date | None] = mapped_column(Date) # MeSH 数据版本日期。通常=date_completed（MeSH 是编目最后一步，两者同天）。年更刷新后=新的 date_completed。NULL=无MeSH(in-process/publisher)
|
||
retracted: Mapped[bool] = mapped_column(Boolean, default=False) # 是否被撤稿
|
||
retraction_details: Mapped[dict | None] = mapped_column(JSONB) # 撤稿详情 {ref_type, ref_pmid, ref_source}
|
||
license_info: Mapped[dict | None] = mapped_column(JSONB) # 许可信息 {href, type, text}
|
||
pre_extracted_data: Mapped[dict | None] = mapped_column(JSONB) # Table 1 预提取基线数据
|
||
is_negative_result: Mapped[bool] = mapped_column(Boolean, default=False) # 阴性结果
|
||
rct_detection: Mapped[dict | None] = mapped_column(JSONB) # RCT 检测结果 {is_rct, confidence, source, evidence}
|
||
search_tsv: Mapped[str | None] = mapped_column(TSVECTOR) # 全文检索向量（PG tsvector）
|
||
author_names_text: Mapped[str | None] = mapped_column(Text) # 从 authors JSONB 提取的 family 文本，由触发器维护，供 trgm 索引
|
||
negative_result_details: Mapped[dict | None] = mapped_column(JSONB) # 阴性结果详情
|
||
ai_summary: Mapped[dict | None] = mapped_column(JSONB) # AI 摘要 {one_liner, structured, implication}
|
||
is_preprint: Mapped[bool] = mapped_column(Boolean, default=False) # 是否为预印本
|
||
auid_data: Mapped[dict | None] = mapped_column(JSONB) # Author Identifiers [{"type": "ORCID", "value": "0000-...", "author_index": 0}]
|
||
cois_statement: Mapped[str | None] = mapped_column(Text) # Conflict of Interest Statement（来自 <CoiStatement>）
|
||
vernacular_title: Mapped[str | None] = mapped_column(Text) # Transliterated/Vernacular Title（来自 <VernacularTitle>）
|
||
|
||
# ═══ PubMed 补充解析字段 ═══
|
||
chemical_list: Mapped[dict] = mapped_column(JSONB, default=list) # ChemicalList [{name, registry_number, mesh_ui}]
|
||
investigators: Mapped[dict] = mapped_column(JSONB, default=list) # InvestigatorList [{family, given, affiliation, identifiers}]
|
||
personal_name_subjects: Mapped[dict] = mapped_column(JSONB, default=list) # PersonalNameSubjectList [{family, given}]
|
||
publication_notes: Mapped[dict] = mapped_column(JSONB, default=list) # PubmedData/PublicationNote ["note1", "note2"]
|
||
gene_symbols: Mapped[dict] = mapped_column(JSONB, default=list) # GeneSymbolList
|
||
num_refs: Mapped[int | None] = mapped_column(Integer) # NumberOfReferences
|
||
publication_status: Mapped[str | None] = mapped_column(String(30)) # PublicationStatus (epublish/ppublish/aheadofprint)
|
||
article_date: Mapped[date | None] = mapped_column(Date) # 电子出版日期（在线先发，早于纸质版）【PubMed: ArticleDate】
|
||
create_date: Mapped[date | None] = mapped_column(Date) # PubMed 记录创建日期（纯日期）【PubMed: CRDT】
|
||
entrez_date: Mapped[date | None] = mapped_column(Date) # PubMed 收录日期（纯日期）【PubMed: EDAT】
|
||
databank_list: Mapped[dict] = mapped_column(JSONB, default=list) # DataBankList [{databank_name, accession_numbers}]
|
||
suppl_mesh_list: Mapped[dict] = mapped_column(JSONB, default=list) # SupplMeshList [{descriptor, ui, type}]
|
||
pharmacological_actions: Mapped[dict] = mapped_column(JSONB, default=list) # PharmacologicalAction [{name, ui}]
|
||
|
||
tag_ids: Mapped[list | None] = mapped_column(ARRAY(Uuid)) # 标签 ID 数组 + GIN，免 JOIN global_literature_tags
|
||
|
||
source: Mapped[str] = mapped_column(String(30), default="pubmed_ftp")
|
||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||
|
||
__table_args__ = (
|
||
# 时序数据 BRIN 代替 B-tree：pub_date/pub_year 物理相关性高，BRIN 体积小两个数量级
|
||
Index("ix_gl_pub_date_brin", "pub_date", postgresql_using="brin"),
|
||
Index("ix_gl_pub_year_brin", "pub_year", postgresql_using="brin"),
|
||
|
||
# Covering index：date 排序 + 高频引用列（index-only scan，排除大 TEXT/JSONB）
|
||
Index("ix_gl_pub_date_covering", "pub_date", "id",
|
||
postgresql_include={"journal_issn", "cited_by_count", "is_oa", "retracted",
|
||
"is_negative_result", "is_preprint", "journal", "pub_year",
|
||
"article_date", "doi", "pmc_id", "language", "citation_status"}),
|
||
|
||
# 高频筛选 partial index
|
||
Index("ix_gl_retracted_true", "retracted", postgresql_where=text("retracted = TRUE")),
|
||
Index("ix_gl_is_oa_true", "is_oa", postgresql_where=text("is_oa = TRUE")),
|
||
Index("ix_gl_is_negative_true", "is_negative_result", postgresql_where=text("is_negative_result = TRUE")),
|
||
Index("ix_gl_is_preprint_true", "is_preprint", postgresql_where=text("is_preprint = TRUE")),
|
||
|
||
# 反范式 tag_ids GIN 索引：标签筛选免 JOIN global_literature_tags
|
||
Index("ix_gl_tag_ids_gin", "tag_ids", postgresql_using="gin"),
|
||
|
||
# 保留旧 B-tree 兼容（迁移后旧索引下线，IR 移除时机：所有实例完成 migration）
|
||
Index("ix_gl_pub_date", "pub_date"),
|
||
Index("ix_gl_pub_year", "pub_year"),
|
||
Index("ix_gl_journal_issn", "journal_issn"),
|
||
Index("ix_gl_created", "created_at"),
|
||
Index("ix_gl_pmc_id", "pmc_id"),
|
||
Index("ix_gl_search_tsv", "search_tsv", postgresql_using="gin"),
|
||
# 筛选性能：布尔/枚举列在纯筛选（无文本搜索）时避免顺序扫描
|
||
Index("ix_gl_retracted", "retracted"),
|
||
Index("ix_gl_is_oa", "is_oa"),
|
||
Index("ix_gl_is_negative", "is_negative_result"),
|
||
Index("ix_gl_is_preprint", "is_preprint"),
|
||
Index("ix_gl_citation_status", "citation_status"),
|
||
# JSONB 筛选索引：Species/Sex/Age 通过 mesh_headings @> 查询
|
||
Index("ix_gl_mesh_headings", "mesh_headings", postgresql_using="gin"),
|
||
# PubMed [TA] 搜索：journal ILIKE + journal_iso ILIKE 兜底
|
||
Index("ix_global_literature_journal_iso_trgm", "journal_iso", postgresql_using="gin", postgresql_ops={"journal_iso": "gin_trgm_ops"}),
|
||
# ILIKE 性能索引（pg_trgm）：title/abstract/journal 用于字段搜索和全文兜底
|
||
Index("ix_global_literature_title_trgm", "title", postgresql_using="gin", postgresql_ops={"title": "gin_trgm_ops"}),
|
||
Index("ix_global_literature_abstract_trgm", "abstract", postgresql_using="gin", postgresql_ops={"abstract": "gin_trgm_ops"}),
|
||
Index("ix_global_literature_journal_trgm", "journal", postgresql_using="gin", postgresql_ops={"journal": "gin_trgm_ops"}),
|
||
# Author 搜索：author_names_text 列 + trgm 索引
|
||
Index("ix_gl_author_names_trgm", "author_names_text", postgresql_using="gin", postgresql_ops={"author_names_text": "gin_trgm_ops"}),
|
||
# 常用筛选列 B-tree
|
||
Index("ix_gl_doi", "doi"),
|
||
Index("ix_gl_journal_iso", "journal_iso"),
|
||
Index("ix_gl_language", "language"),
|
||
# JSONB @> 查询索引：pub_type/grant/chemical/study_design 筛选
|
||
Index("ix_gl_pub_types_gin", "pub_types", postgresql_using="gin"),
|
||
Index("ix_gl_study_design_gin", "study_design", postgresql_using="gin"),
|
||
Index("ix_gl_grants_gin", "grants", postgresql_using="gin"),
|
||
Index("ix_gl_chemical_list_gin", "chemical_list", postgresql_using="gin"),
|
||
)
|
||
|
||
|
||
class GlobalTag(Base):
|
||
__tablename__ = "global_tags"
|
||
|
||
id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=new_uuid)
|
||
mesh_ui: Mapped[str | None] = mapped_column(String(20))
|
||
name_zh: Mapped[str | None] = mapped_column(String(200))
|
||
name_en: Mapped[str | None] = mapped_column(String(200))
|
||
path: Mapped[str] = mapped_column(String(500), nullable=False)
|
||
level: Mapped[int] = mapped_column(Integer, default=1)
|
||
parent_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, ForeignKey("global_tags.id", ondelete="CASCADE"))
|
||
tree_number: Mapped[str | None] = mapped_column(String(100))
|
||
entry_terms: Mapped[dict | None] = mapped_column(JSONB) # MeSH 入口词列表（~200k，从 NLM desc2025.asc 导入）
|
||
tag_category: Mapped[str] = mapped_column(String(30), default="cancer")
|
||
sort_order: Mapped[int] = mapped_column(Integer, default=0)
|
||
icon: Mapped[str | None] = mapped_column(String(50))
|
||
is_selectable: Mapped[bool] = mapped_column(Boolean, default=True)
|
||
article_count: Mapped[int] = mapped_column(Integer, default=0)
|
||
source: Mapped[str] = mapped_column(String(20), default="auto")
|
||
# "manual" = 手工维护（种子数据 / 后台新增）
|
||
# "mesh" = C04 MeSH 批量导入（NLM 权威来源）
|
||
# "auto" = 管道懒创建（待审核）
|
||
is_active: Mapped[bool] = mapped_column(Boolean, default=False)
|
||
# True = 已确认，对用户可见
|
||
# False = 未确认，仅后台可见
|
||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||
|
||
__table_args__ = (
|
||
Index("ix_gt_category", "tag_category"),
|
||
Index("ix_gt_mesh_ui", "mesh_ui"),
|
||
Index("ix_gt_path", "path"),
|
||
)
|
||
|
||
|
||
class GlobalLiteratureTag(Base):
|
||
__tablename__ = "global_literature_tags"
|
||
|
||
literature_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("global_literature.id", ondelete="CASCADE"), primary_key=True)
|
||
tag_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("global_tags.id", ondelete="CASCADE"), primary_key=True)
|
||
is_major: Mapped[bool] = mapped_column(Boolean, default=False)
|
||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||
|
||
__table_args__ = (
|
||
Index("ix_glt_tag", "tag_id"),
|
||
Index("ix_glt_literature", "literature_id"),
|
||
)
|
||
|
||
|
||
class GlobalTagTreeNumber(Base):
|
||
"""MeSH Descriptor → TreeNumber 多对多映射
|
||
|
||
一个 MeSH Descriptor（global_tags）可以有多个树号（如 C04.588.894.797.520）。
|
||
用于 [MH] 搜索展开（tree_number LIKE 前缀匹配）和 C04 肿瘤分类判定。
|
||
"""
|
||
__tablename__ = "global_tag_tree_numbers"
|
||
|
||
id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=new_uuid)
|
||
tag_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("global_tags.id", ondelete="CASCADE"), nullable=False)
|
||
tree_number: Mapped[str] = mapped_column(String(100), nullable=False)
|
||
tree_category: Mapped[str] = mapped_column(String(10), nullable=False) # C04, C08, C15, ...
|
||
is_primary: Mapped[bool] = mapped_column(Boolean, default=False)
|
||
|
||
__table_args__ = (
|
||
Index("ix_gttn_tag_id", "tag_id"),
|
||
Index("ix_gttn_tree_category", "tree_category"),
|
||
Index("ix_gttn_tree_number", "tree_number"),
|
||
UniqueConstraint("tag_id", "tree_number", name="uq_gttn_tag_tree"),
|
||
)
|
||
|
||
|
||
class GlobalJournal(Base):
|
||
__tablename__ = "global_journals"
|
||
|
||
id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, default=new_uuid)
|
||
name: Mapped[str] = mapped_column(String(500), nullable=False)
|
||
issn: Mapped[str | None] = mapped_column(String(20))
|
||
eissn: Mapped[str | None] = mapped_column(String(20))
|
||
tier: Mapped[str] = mapped_column(String(5), default="4")
|
||
impact_factor: Mapped[float | None] = mapped_column()
|
||
publisher: Mapped[str | None] = mapped_column(String(300))
|
||
article_count: Mapped[int] = mapped_column(Integer, default=0)
|
||
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
||
priority_score: Mapped[float | None] = mapped_column() # 综合优先级评分 0-100
|
||
priority_source: Mapped[str | None] = mapped_column(String(20), default="auto") # 'auto' | 'manual'
|
||
specialty: Mapped[str | None] = mapped_column(String(50)) # 'oncology' | NULL
|
||
nlm_subsets: Mapped[list[str] | None] = mapped_column(ARRAY(String(30)), default=list) # NLM 期刊子集（AIM/D/N/S 等）
|
||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||
|
||
__table_args__ = (
|
||
Index("ix_gj_tier", "tier"),
|
||
Index("ix_gj_priority", "priority_score", postgresql_using="btree"),
|
||
UniqueConstraint("issn", name="uq_gj_issn"),
|
||
UniqueConstraint("name", name="uq_gj_name"),
|
||
)
|