416 lines
17 KiB
Bash
416 lines
17 KiB
Bash
#!/usr/bin/env bash
|
||
# ═══════════════════════════════════════════════════════════════════
|
||
# deploy.sh — SciLit 生产一键部署（docker-compose.prod.yml）
|
||
#
|
||
# 用法:
|
||
# bash deploy/deploy.sh # 正式部署（在服务器 /root/scilit 下执行）
|
||
# bash deploy/deploy.sh --dry-run # 只打印将执行的命令，不执行任何变更
|
||
# bash deploy/deploy.sh --yes # 跳过部署前 y/N 确认提示
|
||
# bash deploy/deploy.sh --help
|
||
#
|
||
# 前置条件:
|
||
# - 在 git 仓库根目录的 deploy/ 下执行（自动定位 REPO_ROOT 与 .env）
|
||
# - 基础设施（postgres/redis/elasticsearch/minio）已在跑（脚本内置健康门）
|
||
# - 生产部署期间勿并发（脚本自带 flock 单实例锁）
|
||
#
|
||
# 流程（对应 docs/16 §2 设计）:
|
||
# 遗留标记检查 → source .env → flock → 工作区校验 → 基础设施健康门 →
|
||
# PREV_SHA → git pull → destructive 判定 → 部署预览+确认 → 迁移前快照 →
|
||
# build + 双 tag → .pending-deploy → 迁移(退出码门控) → up -d --no-deps →
|
||
# 健康轮询 → versions.log → 镜像治理
|
||
#
|
||
# 交互与进度:
|
||
# · 每个阶段打印 [n/11] 标题 + 完成耗时，随时知道进行到哪
|
||
# · 静默长命令（快照/迁移/容器切换）带每秒心跳，不会"看起来假死"
|
||
# · 部署前打印预览并 y/N 确认（--yes 跳过；非交互终端自动继续）
|
||
#
|
||
# 失败处理分两段:
|
||
# · 迁移前失败（快照/构建阶段）→ 容器仍是旧的，只清理中间态，不触发回滚
|
||
# · 迁移已跑/容器已切后失败 → 调 rollback.sh --auto（读本次 .pending-deploy）
|
||
# ═══════════════════════════════════════════════════════════════════
|
||
set -euo pipefail
|
||
|
||
DEPLOY_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||
REPO_ROOT="$(cd "$DEPLOY_DIR/.." && pwd)"
|
||
cd "$REPO_ROOT"
|
||
|
||
COMPOSE_FILE="docker-compose.prod.yml"
|
||
COMPOSE=(docker compose -f "$COMPOSE_FILE")
|
||
BACKUP_DIR="${BACKUP_DIR:-/data/backups}"
|
||
KEEP_TAGS=10 # 保留最近 N 个带 tag 版本镜像（§5 count 制）
|
||
HEALTH_POLLS=12 # 健康轮询次数 × 5s = 60s 上限
|
||
|
||
GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'; BLUE='\033[0;34m'; NC='\033[0m'
|
||
info() { echo -e "${GREEN}[✓]${NC} $1"; }
|
||
warn() { echo -e "${YELLOW}[!]${NC} $1"; }
|
||
error() { echo -e "${RED}[✗]${NC} $1"; exit 1; }
|
||
|
||
# ── 进度框架：阶段计数 + 耗时 + 长命令心跳 ──
|
||
STAGE_TOTAL=11
|
||
STAGE_N=0
|
||
stage() {
|
||
STAGE_N=$((STAGE_N+1))
|
||
STAGE_START=$(date +%s)
|
||
echo
|
||
echo -e "${BLUE}════ [$STAGE_N/$STAGE_TOTAL] $1 ════${NC}"
|
||
}
|
||
stage_done() {
|
||
echo -e "${GREEN}──── 阶段 $STAGE_N 完成（$(( $(date +%s) - STAGE_START ))s）${NC}"
|
||
}
|
||
|
||
TICKER_PID=""
|
||
# 长命令心跳：每秒刷新耗时，防止 pg_dump / 迁移这类静默命令"看起来假死"
|
||
ticker_start() {
|
||
[ -t 1 ] || return 0 # 非 TTY（日志重定向）不跑心跳
|
||
[ "$DRY_RUN" -eq 1 ] && return 0
|
||
local msg="$1" t=0
|
||
( while :; do t=$((t+1)); printf "\r${YELLOW}%s … 已 %ss（无输出属正常）${NC}" "$msg" "$t"; sleep 1; done ) &
|
||
TICKER_PID=$!
|
||
}
|
||
ticker_stop() {
|
||
[ -n "$TICKER_PID" ] || return 0
|
||
kill "$TICKER_PID" 2>/dev/null || true
|
||
wait "$TICKER_PID" 2>/dev/null || true
|
||
TICKER_PID=""
|
||
printf "\r\033[K" # 清掉心跳行，避免与下条输出混行
|
||
}
|
||
cleanup() { ticker_stop; }
|
||
trap cleanup EXIT
|
||
|
||
DRY_RUN=0; YES=0
|
||
# 所有变更命令统一走 run()：--dry-run 下只打印不执行
|
||
run() {
|
||
if [ "$DRY_RUN" -eq 1 ]; then
|
||
echo -e "${YELLOW}[DRY-RUN]${NC} $*"
|
||
return 0
|
||
fi
|
||
"$@"
|
||
}
|
||
|
||
usage() {
|
||
cat <<'EOF'
|
||
用法: bash deploy/deploy.sh [--dry-run] [--yes] [--help]
|
||
|
||
--dry-run 只打印将执行的命令，不执行任何变更（SHA 显示当前状态）
|
||
--yes 跳过部署前的 y/N 确认提示（非交互终端本就会自动继续）
|
||
--help 显示本帮助
|
||
EOF
|
||
}
|
||
|
||
for arg in "$@"; do
|
||
case "$arg" in
|
||
--dry-run) DRY_RUN=1 ;;
|
||
--yes) YES=1 ;;
|
||
--help) usage; exit 0 ;;
|
||
*) usage; exit 1 ;;
|
||
esac
|
||
done
|
||
|
||
# ═══ [1/11] 前置检查 ═══
|
||
stage "前置检查（遗留标记 / .env / 单实例锁 / 工作区）"
|
||
|
||
# 1. 遗留 .pending-deploy 检测（I：上次部署中途崩溃的残留，不静默覆盖）
|
||
if [ -f "$DEPLOY_DIR/.pending-deploy" ]; then
|
||
error "⚠️ 上一次部署异常退出，残留 ${DEPLOY_DIR}/.pending-deploy。
|
||
请先确认当前状态（容器/迁移/versions.log）再继续，然后删除该标记或人工处理后重跑。"
|
||
fi
|
||
|
||
# 2. source .env（G：PG_PASSWORD/REDIS_PASSWORD 供 pg_dump 备选与探活）
|
||
if [ ! -f "$REPO_ROOT/.env" ]; then
|
||
error "未找到 ${REPO_ROOT}/.env（含密钥，不进 git）。请先在服务器配置。"
|
||
fi
|
||
set -a; source "$REPO_ROOT/.env"; set +a
|
||
|
||
# 3. .env 复杂类型预检（fail fast）：pydantic-settings 会把 list/dict 类型字段按 JSON 解析，
|
||
# 写成了逗号列表会在迁移时炸（8-10 事故），这里提前捕获，给出正确写法提示。
|
||
# 仅检查 config.py 中的复杂类型字段（现仅 CORS_ORIGINS）；str 类型（如 REDIS_URL）
|
||
# 不按 JSON 解析，预检会误报，不要加进来。
|
||
# 注意：必须从 .env 文件直接 grep 原始值——bash `source .env` 会把 `KEY=["a","b"]`
|
||
# 按数组语法解析并剥掉引号，${KEY} 拿到的是非法 JSON，造成假阴性。
|
||
json_env_check() {
|
||
local var="$1" raw
|
||
raw="$(grep -E "^[[:space:]]*(export[[:space:]]+)?${var}[[:space:]]*=" "$REPO_ROOT/.env" | tail -1 | sed -E "s/^[^=]*=//" || true)"
|
||
[ -z "$raw" ] && return 0
|
||
if ! python3 -c "import json,sys; json.loads(sys.argv[1])" "$raw" 2>/dev/null; then
|
||
error "$var 必须是合法 JSON（如 [\"https://a.com\"]），.env 当前值: $raw
|
||
请修改 $REPO_ROOT/.env 后重试。"
|
||
fi
|
||
}
|
||
json_env_check CORS_ORIGINS
|
||
|
||
# 3. flock 单实例锁（防两人/两终端并发部署）
|
||
exec 9>/tmp/scilit-deploy.lock
|
||
flock -n 9 || error "已有部署/回滚在进行中（/tmp/scilit-deploy.lock 被锁）。"
|
||
|
||
# 4. 工作区校验（防 hotfix 残留导致 git pull 冲突）
|
||
# 只拦「已跟踪文件的改动」（这才是 pull 冲突来源）；未跟踪文件（如服务器运维
|
||
# 工具 backup.sh/tmp_*.py）warn 不拦
|
||
STALE_CHANGES="$(git status --porcelain | grep -v '^??' || true)"
|
||
if [ -n "$STALE_CHANGES" ]; then
|
||
error "工作区有已跟踪文件改动（可能为 hotfix 残留，git pull 会冲突）：
|
||
$STALE_CHANGES"
|
||
fi
|
||
UNTRACKED="$(git status --porcelain | grep '^??' || true)"
|
||
if [ -n "$UNTRACKED" ]; then
|
||
warn "存在未跟踪文件（不影响 pull，仅当新代码含同名文件时需注意）：
|
||
$UNTRACKED"
|
||
fi
|
||
stage_done
|
||
|
||
# ═══ [2/11] 基础设施健康门 ═══
|
||
stage "基础设施健康门（postgres/redis/elasticsearch/minio）"
|
||
infra_ok() {
|
||
if [ "$DRY_RUN" -eq 1 ]; then
|
||
echo "[DRY-RUN] 健康门：postgres/redis/elasticsearch/minio 模拟通过"
|
||
return 0
|
||
fi
|
||
local svc cid health status
|
||
for svc in postgres redis elasticsearch minio; do
|
||
cid=$("${COMPOSE[@]}" ps -q "$svc" 2>/dev/null || true)
|
||
if [ -z "$cid" ]; then
|
||
error "基础设施 $svc 未运行。请先冷启动：
|
||
docker compose -f $COMPOSE_FILE up -d postgres redis elasticsearch minio gitea"
|
||
fi
|
||
health=$(docker inspect --format '{{.State.Health.Status}}' "$cid" 2>/dev/null || true)
|
||
status=$(docker inspect --format '{{.State.Status}}' "$cid" 2>/dev/null || true)
|
||
if [ -n "$health" ]; then
|
||
[ "$health" = "healthy" ] || error "基础设施 $svc health=$health（期望 healthy）。请先确认状态再部署。"
|
||
else
|
||
[ "$status" = "running" ] || error "基础设施 $svc status=$status（无 healthcheck，期望 running）。"
|
||
fi
|
||
done
|
||
info "基础设施健康门通过"
|
||
}
|
||
infra_ok
|
||
stage_done
|
||
|
||
# ═══ [3/11] 拉取代码 ═══
|
||
stage "拉取代码（PREV_SHA → NEW_SHA）"
|
||
# PREV_SHA（F：destructive 判定基线，必须在 git pull 之前）
|
||
PREV_SHA="$(git rev-parse HEAD)"
|
||
info "当前生产版本: $PREV_SHA"
|
||
ticker_start "git pull（网络慢时可能卡住，属正常）"
|
||
run git pull
|
||
ticker_stop
|
||
NEW_SHA="$(git rev-parse --short HEAD)"
|
||
info "目标版本: $NEW_SHA"
|
||
stage_done
|
||
|
||
# ═══ [4/11] 迁移影响评估 ═══
|
||
stage "迁移影响评估（destructive 判定）"
|
||
detect_destructive() {
|
||
local prev="$1" new="$2" files f fn
|
||
files="$(git diff --name-only "$prev" "$new" -- alembic/versions/)"
|
||
[ -z "$files" ] && { echo "否"; return 0; }
|
||
for f in $files; do
|
||
fn="$(basename "$f")"
|
||
case "$fn" in
|
||
destructive_*) echo "是"; return 0 ;;
|
||
esac
|
||
if grep -q '# DESTRUCTIVE' "$REPO_ROOT/$f" 2>/dev/null; then echo "是"; return 0; fi
|
||
done
|
||
if git diff "$prev" "$new" -- alembic/versions/ | grep '^+' | grep -qE \
|
||
'op\.drop_(table|column|index)|op\.rename_(table|column|index)|op\.alter_column|existing_type|op\.execute[^)]*(DROP|DELETE|UPDATE|TRUNCATE)'; then
|
||
echo "是"; return 0
|
||
fi
|
||
echo "否"
|
||
}
|
||
DESTRUCTIVE="$(detect_destructive "$PREV_SHA" "$(git rev-parse HEAD)")"
|
||
if [ "$DESTRUCTIVE" = "是" ]; then
|
||
warn "本次迁移含破坏性操作——回滚时将用 predeploy 快照恢复数据（见 docs/16 §3）"
|
||
else
|
||
info "本次迁移 non-destructive（回滚仅切镜像，无需数据回退）"
|
||
fi
|
||
stage_done
|
||
|
||
# ═══ [5/11] 部署确认 ═══
|
||
stage "部署确认"
|
||
SNAPSHOT="$BACKUP_DIR/predeploy_$(date +%Y%m%d_%H%M%S).dump"
|
||
echo
|
||
echo -e "${BLUE}┌────────────────── 部署预览 ──────────────────┐${NC}"
|
||
echo -e "${BLUE}│${NC} 当前生产版本 : ${PREV_SHA:0:12}"
|
||
echo -e "${BLUE}│${NC} 目标版本 : $NEW_SHA"
|
||
if [ "$DESTRUCTIVE" = "是" ]; then
|
||
echo -e "${BLUE}│${NC} 迁移破坏性 : ${RED}是${NC}（回滚走快照恢复）"
|
||
else
|
||
echo -e "${BLUE}│${NC} 迁移破坏性 : ${GREEN}否${NC}（回滚仅切镜像）"
|
||
fi
|
||
echo -e "${BLUE}│${NC} predeploy 快照: $SNAPSHOT"
|
||
echo -e "${BLUE}│${NC} 之后动作 : alembic upgrade head → up -d --no-deps backend worker frontend"
|
||
echo -e "${BLUE}└──────────────────────────────────────────────┘${NC}"
|
||
echo
|
||
if [ "$DRY_RUN" -eq 1 ]; then
|
||
info "dry-run 模式，跳过确认"
|
||
elif [ "$YES" -eq 1 ]; then
|
||
info "已用 --yes 跳过确认，直接开始"
|
||
elif [ ! -t 0 ]; then
|
||
info "非交互终端（stdin 非 TTY），自动继续"
|
||
else
|
||
if read -r -p "确认开始部署？(y/N) " yn; then
|
||
[ "$yn" = "y" ] || [ "$yn" = "Y" ] || error "已取消，未做任何改动。"
|
||
else
|
||
error "已取消（输入中断）。"
|
||
fi
|
||
fi
|
||
stage_done
|
||
|
||
# ═══ [6/11] 迁移前快照 ═══
|
||
stage "迁移前快照 + 清理"
|
||
get_db_rev() {
|
||
if [ "$DRY_RUN" -eq 1 ]; then echo "-"; return 0; fi
|
||
"${COMPOSE[@]}" exec -T postgres psql -U scilit -d scilit -tAc \
|
||
"SELECT version_num FROM alembic_version" 2>/dev/null | tr -d '[:space:]' | cut -c1-12
|
||
}
|
||
# before alembic head（空则记 '-'，首次部署容错）
|
||
ALEMBIC_BEFORE="$(get_db_rev || true)"
|
||
info "迁移前 alembic head: ${ALEMBIC_BEFORE:-（空/首次）}"
|
||
|
||
mkdir -p "$BACKUP_DIR"
|
||
info "predeploy 全量快照（pg_dump -Fc；库大时 1–3 分钟，期间无输出属正常）"
|
||
ticker_start "predeploy 快照生成中"
|
||
if [ "$DRY_RUN" -eq 1 ]; then
|
||
echo -e "${YELLOW}[DRY-RUN]${NC} pg_dump → $SNAPSHOT"
|
||
else
|
||
PGPASSWORD="${PG_PASSWORD}" "${COMPOSE[@]}" exec -T postgres \
|
||
pg_dump -Fc --no-owner --no-privileges -U scilit -d scilit > "$SNAPSHOT"
|
||
if [ ! -s "$SNAPSHOT" ]; then
|
||
rm -f "$SNAPSHOT"
|
||
error "predeploy 快照生成失败（空文件），中止部署。"
|
||
fi
|
||
fi
|
||
ticker_stop
|
||
info "predeploy 快照: $SNAPSHOT"
|
||
|
||
# predeploy 快照保留最近 N=3（E1：count 判定 + sort | head -n -3 | xargs -r rm）
|
||
prune_snapshots() {
|
||
if [ "$DRY_RUN" -eq 1 ]; then return 0; fi
|
||
local count
|
||
count="$(ls "$BACKUP_DIR"/predeploy_*.dump 2>/dev/null | wc -l)"
|
||
if [ "$count" -gt 3 ]; then
|
||
ls "$BACKUP_DIR"/predeploy_*.dump | sort | head -n -3 | xargs -r rm -f
|
||
info "已清理旧 predeploy 快照（保留最近 3 个）"
|
||
fi
|
||
}
|
||
prune_snapshots
|
||
stage_done
|
||
|
||
# ═══ [7/11] 构建镜像 + 双 tag ═══
|
||
stage "构建镜像（backend/frontend → $NEW_SHA）"
|
||
if [ "$PREV_SHA" = "$(git rev-parse HEAD)" ] && [ -z "$(git diff --name-only "$PREV_SHA" -- docker-compose.prod.yml deploy)" ]; then
|
||
warn "代码无更新（HEAD 未变），执行幂等重部署"
|
||
fi
|
||
export BACKEND_TAG="$NEW_SHA" FRONTEND_TAG="$NEW_SHA"
|
||
info "build 前 export 双 tag，compose build 直接打到 scilit/backend:${NEW_SHA} / scilit/frontend:${NEW_SHA}"
|
||
run "${COMPOSE[@]}" build backend frontend
|
||
stage_done
|
||
|
||
# ═══ [8/11] 数据库迁移 ═══
|
||
stage "数据库迁移（heads 预检 + alembic upgrade head）"
|
||
# 写 .pending-deploy（N1：快照后、迁移前；失败自动回滚唯一数据源）
|
||
if [ "$DRY_RUN" -ne 1 ]; then
|
||
echo "$NEW_SHA $NEW_SHA $DESTRUCTIVE $SNAPSHOT" > "$DEPLOY_DIR/.pending-deploy"
|
||
fi
|
||
info "写入部署标记: $DEPLOY_DIR/.pending-deploy"
|
||
|
||
check_single_head() {
|
||
if [ "$DRY_RUN" -eq 1 ]; then return 0; fi
|
||
local n
|
||
n="$("${COMPOSE[@]}" run --no-deps --rm backend alembic -c alembic/alembic.ini heads 2>/dev/null | grep -cE '[0-9a-f]{12}\b' || true)"
|
||
[ "$n" -le 1 ] || error "alembic 多 head（$n 个），请先合并（alembic merge 或串行 rebase）。"
|
||
}
|
||
check_single_head
|
||
|
||
auto_rollback() {
|
||
echo -e "${RED}[✗] 部署部分完成（迁移已跑/容器已切），触发自动回滚${NC}"
|
||
bash "$DEPLOY_DIR/rollback.sh" --auto || warn "自动回滚失败，请人工介入（详见 docs/16 §3 回滚 runbook）"
|
||
exit 1
|
||
}
|
||
|
||
info "执行迁移: alembic upgrade head（已到 head 时无输出属正常）"
|
||
ticker_start "数据库迁移中"
|
||
if ! run "${COMPOSE[@]}" run --no-deps --rm backend alembic -c alembic/alembic.ini upgrade head; then
|
||
ticker_stop
|
||
auto_rollback
|
||
fi
|
||
ticker_stop
|
||
|
||
ALEMBIC_AFTER="$(get_db_rev || true)"
|
||
info "迁移后 alembic head: ${ALEMBIC_AFTER:-（空/首次）}"
|
||
stage_done
|
||
|
||
# ═══ [9/11] 切换容器 + 健康轮询 ═══
|
||
stage "切换容器（up -d --no-deps backend worker frontend）+ 健康轮询"
|
||
info "切换容器到新版本（backend worker frontend）"
|
||
ticker_start "切换容器中"
|
||
if ! run "${COMPOSE[@]}" up -d --no-deps backend worker frontend; then
|
||
ticker_stop
|
||
auto_rollback
|
||
fi
|
||
ticker_stop
|
||
|
||
wait_backend_healthy() {
|
||
local i
|
||
for i in $(seq 1 "$HEALTH_POLLS"); do
|
||
if "${COMPOSE[@]}" exec -T backend curl -sf http://localhost:8000/health >/dev/null 2>&1; then
|
||
return 0
|
||
fi
|
||
echo -e "${YELLOW}[$i/$HEALTH_POLLS] backend 尚未就绪，5s 后重试…${NC}"
|
||
sleep 5
|
||
done
|
||
return 1
|
||
}
|
||
check_frontend() {
|
||
"${COMPOSE[@]}" exec -T frontend wget -q -O- http://localhost/ >/dev/null 2>&1
|
||
}
|
||
|
||
if [ "$DRY_RUN" -eq 1 ]; then
|
||
info "[DRY-RUN] 健康轮询：backend /health + frontend 200（模拟通过）"
|
||
elif wait_backend_healthy; then
|
||
info "backend /health 通过"
|
||
if check_frontend; then
|
||
info "frontend 200 通过"
|
||
else
|
||
warn "frontend 健康检查未过，但 backend 已就绪——请人工确认前端状态"
|
||
fi
|
||
else
|
||
auto_rollback
|
||
fi
|
||
stage_done
|
||
|
||
# ═══ [10/11] 记录版本 ═══
|
||
stage "记录版本（versions.log + 清理部署标记）"
|
||
if [ "$DRY_RUN" -eq 1 ]; then
|
||
echo -e "${YELLOW}[DRY-RUN]${NC} 追加 versions.log + 删除 .pending-deploy"
|
||
else
|
||
echo "DEPLOY $NEW_SHA $NEW_SHA $DESTRUCTIVE ${ALEMBIC_BEFORE:-} ${ALEMBIC_AFTER:-} $(date '+%Y-%m-%d %H:%M:%S %z')" \
|
||
>> "$DEPLOY_DIR/versions.log"
|
||
tail -n 200 "$DEPLOY_DIR/versions.log" > "$DEPLOY_DIR/versions.log.tmp"
|
||
mv "$DEPLOY_DIR/versions.log.tmp" "$DEPLOY_DIR/versions.log"
|
||
rm -f "$DEPLOY_DIR/.pending-deploy"
|
||
fi
|
||
info "部署成功，已写入 versions.log"
|
||
stage_done
|
||
|
||
# ═══ [11/11] 镜像治理 ═══
|
||
stage "镜像治理（保留最近 $KEEP_TAGS 个带 tag 版本 + 清理 dangling）"
|
||
prune_images() {
|
||
if [ "$DRY_RUN" -eq 1 ]; then return 0; fi
|
||
local repo count to_remove t
|
||
for repo in scilit/backend scilit/frontend; do
|
||
count="$(docker images "$repo" --format '{{.Tag}}' | grep -v '^latest$' | wc -l)"
|
||
if [ "$count" -gt "$KEEP_TAGS" ]; then
|
||
to_remove="$(docker images "$repo" --format '{{.Tag}} {{.CreatedAt}}' \
|
||
| grep -v '^latest ' | sort -k2 | head -n $((count - KEEP_TAGS)) | awk '{print $1}')"
|
||
for t in $to_remove; do
|
||
docker rmi "$repo:$t" >/dev/null 2>&1 || true
|
||
done
|
||
info "已清理 $repo 旧版本镜像（保留最近 $KEEP_TAGS 个）"
|
||
fi
|
||
done
|
||
docker image prune --filter "until=168h" -f >/dev/null 2>&1 || true
|
||
}
|
||
prune_images
|
||
stage_done
|
||
|
||
info "全部完成 ✅ 当前版本: $NEW_SHA"
|