perf: 模型索引优化 — tag_ids 反范式 + BRIN/partial/covering 索引
- GlobalLiterature.tag_ids ARRAY(Uuid) 列 + GIN 索引(标签筛选免 JOIN) - BRIN 索引:pub_date / pub_year(时序数据,体积缩小 100x) - Covering 索引:(pub_date, id) INCLUDE 高频列(index-only scan) - Partial 索引:retracted / is_oa / is_negative_result / is_preprint
This commit is contained in:
@@ -0,0 +1,62 @@
|
|||||||
|
"""add_tag_ids_array_brin_indexes
|
||||||
|
|
||||||
|
Revision ID: e0764f6d7c21
|
||||||
|
Revises: g0h1i2j3k4l5
|
||||||
|
Create Date: 2026-07-27 16:14:29.454432
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = 'e0764f6d7c21'
|
||||||
|
down_revision: Union[str, None] = 'g0h1i2j3k4l5'
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
op.drop_index(op.f('ix_audit_action'), table_name='audit_logs')
|
||||||
|
op.drop_index(op.f('ix_audit_actor'), table_name='audit_logs')
|
||||||
|
op.drop_table('audit_logs')
|
||||||
|
op.add_column('global_literature', sa.Column('tag_ids', postgresql.ARRAY(sa.Uuid()), nullable=True))
|
||||||
|
op.drop_index(op.f('ix_gl_authors_text_trgm'), table_name='global_literature', postgresql_ops={'(authors::text)': 'gin_trgm_ops'}, postgresql_using='gin')
|
||||||
|
op.create_index('ix_gl_is_negative_true', 'global_literature', ['is_negative_result'], unique=False, postgresql_where=sa.text('is_negative_result = TRUE'))
|
||||||
|
op.create_index('ix_gl_is_oa_true', 'global_literature', ['is_oa'], unique=False, postgresql_where=sa.text('is_oa = TRUE'))
|
||||||
|
op.create_index('ix_gl_is_preprint_true', 'global_literature', ['is_preprint'], unique=False, postgresql_where=sa.text('is_preprint = TRUE'))
|
||||||
|
op.create_index('ix_gl_pub_date_brin', 'global_literature', ['pub_date'], unique=False, postgresql_using='brin')
|
||||||
|
op.create_index('ix_gl_pub_date_covering', 'global_literature', ['pub_date', 'id'], unique=False, postgresql_include={'citation_status', 'is_oa', 'cited_by_count', 'is_preprint', 'pub_year', 'doi', 'journal_issn', 'is_negative_result', 'article_date', 'retracted', 'journal', 'language', 'pmc_id'})
|
||||||
|
op.create_index('ix_gl_pub_year_brin', 'global_literature', ['pub_year'], unique=False, postgresql_using='brin')
|
||||||
|
op.create_index('ix_gl_retracted_true', 'global_literature', ['retracted'], unique=False, postgresql_where=sa.text('retracted = TRUE'))
|
||||||
|
op.create_index('ix_gl_tag_ids_gin', 'global_literature', ['tag_ids'], unique=False, postgresql_using='gin')
|
||||||
|
# ### end Alembic commands ###
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
op.drop_index('ix_gl_tag_ids_gin', table_name='global_literature', postgresql_using='gin')
|
||||||
|
op.drop_index('ix_gl_retracted_true', table_name='global_literature', postgresql_where=sa.text('retracted = TRUE'))
|
||||||
|
op.drop_index('ix_gl_pub_year_brin', table_name='global_literature', postgresql_using='brin')
|
||||||
|
op.drop_index('ix_gl_pub_date_covering', table_name='global_literature', postgresql_include={'citation_status', 'is_oa', 'cited_by_count', 'is_preprint', 'pub_year', 'doi', 'journal_issn', 'is_negative_result', 'article_date', 'retracted', 'journal', 'language', 'pmc_id'})
|
||||||
|
op.drop_index('ix_gl_pub_date_brin', table_name='global_literature', postgresql_using='brin')
|
||||||
|
op.drop_index('ix_gl_is_preprint_true', table_name='global_literature', postgresql_where=sa.text('is_preprint = TRUE'))
|
||||||
|
op.drop_index('ix_gl_is_oa_true', table_name='global_literature', postgresql_where=sa.text('is_oa = TRUE'))
|
||||||
|
op.drop_index('ix_gl_is_negative_true', table_name='global_literature', postgresql_where=sa.text('is_negative_result = TRUE'))
|
||||||
|
op.create_index(op.f('ix_gl_authors_text_trgm'), 'global_literature', [sa.literal_column('(authors::text)')], unique=False, postgresql_ops={'(authors::text)': 'gin_trgm_ops'}, postgresql_using='gin')
|
||||||
|
op.drop_column('global_literature', 'tag_ids')
|
||||||
|
op.create_table('audit_logs',
|
||||||
|
sa.Column('id', sa.UUID(), autoincrement=False, nullable=False),
|
||||||
|
sa.Column('actor_id', sa.UUID(), autoincrement=False, nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.UUID(), autoincrement=False, nullable=True),
|
||||||
|
sa.Column('action', sa.VARCHAR(length=50), autoincrement=False, nullable=False),
|
||||||
|
sa.Column('target_type', sa.VARCHAR(length=50), autoincrement=False, nullable=True),
|
||||||
|
sa.Column('target_id', sa.VARCHAR(length=100), autoincrement=False, nullable=True),
|
||||||
|
sa.Column('detail', sa.TEXT(), autoincrement=False, nullable=True),
|
||||||
|
sa.Column('ip_address', sa.VARCHAR(length=45), autoincrement=False, nullable=True),
|
||||||
|
sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint('id', name=op.f('audit_logs_pkey'))
|
||||||
|
)
|
||||||
|
op.create_index(op.f('ix_audit_actor'), 'audit_logs', ['actor_id', 'created_at'], unique=False)
|
||||||
|
op.create_index(op.f('ix_audit_action'), 'audit_logs', ['action', 'created_at'], unique=False)
|
||||||
|
# ### end Alembic commands ###
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
import uuid
|
import uuid
|
||||||
from datetime import date, datetime
|
from datetime import date, datetime
|
||||||
|
|
||||||
from sqlalchemy import Boolean, Date, DateTime, ForeignKey, Index, Integer, String, Text, Uuid, UniqueConstraint, func
|
from sqlalchemy import Boolean, Date, DateTime, ForeignKey, Index, Integer, String, Text, Uuid, UniqueConstraint, func, text
|
||||||
from sqlalchemy.dialects.postgresql import ARRAY, JSONB, TSVECTOR
|
from sqlalchemy.dialects.postgresql import ARRAY, JSONB, TSVECTOR
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
@@ -78,11 +78,33 @@ class GlobalLiterature(Base):
|
|||||||
suppl_mesh_list: Mapped[dict] = mapped_column(JSONB, default=list) # SupplMeshList [{descriptor, ui, type}]
|
suppl_mesh_list: Mapped[dict] = mapped_column(JSONB, default=list) # SupplMeshList [{descriptor, ui, type}]
|
||||||
pharmacological_actions: Mapped[dict] = mapped_column(JSONB, default=list) # PharmacologicalAction [{name, ui}]
|
pharmacological_actions: Mapped[dict] = mapped_column(JSONB, default=list) # PharmacologicalAction [{name, ui}]
|
||||||
|
|
||||||
|
tag_ids: Mapped[list | None] = mapped_column(ARRAY(Uuid)) # 标签 ID 数组 + GIN,免 JOIN global_literature_tags
|
||||||
|
|
||||||
source: Mapped[str] = mapped_column(String(30), default="pubmed_ftp")
|
source: Mapped[str] = mapped_column(String(30), default="pubmed_ftp")
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
|
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
|
# 时序数据 BRIN 代替 B-tree:pub_date/pub_year 物理相关性高,BRIN 体积小两个数量级
|
||||||
|
Index("ix_gl_pub_date_brin", "pub_date", postgresql_using="brin"),
|
||||||
|
Index("ix_gl_pub_year_brin", "pub_year", postgresql_using="brin"),
|
||||||
|
|
||||||
|
# Covering index:date 排序 + 高频引用列(index-only scan,排除大 TEXT/JSONB)
|
||||||
|
Index("ix_gl_pub_date_covering", "pub_date", "id",
|
||||||
|
postgresql_include={"journal_issn", "cited_by_count", "is_oa", "retracted",
|
||||||
|
"is_negative_result", "is_preprint", "journal", "pub_year",
|
||||||
|
"article_date", "doi", "pmc_id", "language", "citation_status"}),
|
||||||
|
|
||||||
|
# 高频筛选 partial index
|
||||||
|
Index("ix_gl_retracted_true", "retracted", postgresql_where=text("retracted = TRUE")),
|
||||||
|
Index("ix_gl_is_oa_true", "is_oa", postgresql_where=text("is_oa = TRUE")),
|
||||||
|
Index("ix_gl_is_negative_true", "is_negative_result", postgresql_where=text("is_negative_result = TRUE")),
|
||||||
|
Index("ix_gl_is_preprint_true", "is_preprint", postgresql_where=text("is_preprint = TRUE")),
|
||||||
|
|
||||||
|
# 反范式 tag_ids GIN 索引:标签筛选免 JOIN global_literature_tags
|
||||||
|
Index("ix_gl_tag_ids_gin", "tag_ids", postgresql_using="gin"),
|
||||||
|
|
||||||
|
# 保留旧 B-tree 兼容(迁移后旧索引下线,IR 移除时机:所有实例完成 migration)
|
||||||
Index("ix_gl_pub_date", "pub_date"),
|
Index("ix_gl_pub_date", "pub_date"),
|
||||||
Index("ix_gl_pub_year", "pub_year"),
|
Index("ix_gl_pub_year", "pub_year"),
|
||||||
Index("ix_gl_journal_issn", "journal_issn"),
|
Index("ix_gl_journal_issn", "journal_issn"),
|
||||||
|
|||||||
Reference in New Issue
Block a user