34047007@qq.com
a6cd99a4ca
OncoLit: a multi-tenant oncology literature search, feed, and collaboration platform. Built with FastAPI + Vue 3 + PostgreSQL. Includes PubMed pipeline, drug approvals, AI summaries, and systematic review tools.
86 lines
2.5 KiB
Bash
86 lines
2.5 KiB
Bash
#!/bin/bash
|
||
# PubMed 增量更新逐文件下载、验证、导入脚本
|
||
# 用法: ssh到服务器后: nohup bash /tmp/pubmed_update_one_by_one.sh &
|
||
# 日志: tail -f /data/pubmed/import_update_files.log
|
||
|
||
set -euo pipefail
|
||
|
||
STAGING=/data/pubmed/staging
|
||
WORKDIR=/data/pubmed/updatefiles
|
||
IMPORTED=/data/pubmed/updatefiles/imported
|
||
COMPOSE_FILE=/root/scilit/docker-compose.prod.yml
|
||
LOG=/data/pubmed/import_update_files.log
|
||
|
||
mkdir -p "$STAGING" "$WORKDIR" "$IMPORTED"
|
||
|
||
log() {
|
||
echo "[$(TZ=Asia/Shanghai date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG"
|
||
}
|
||
|
||
log "=== 开始逐文件导入：195 个增量文件（1335-1529）==="
|
||
log "目录: $STAGING -> $WORKDIR -> $IMPORTED"
|
||
|
||
GOOD=0
|
||
SKIP=0
|
||
FAIL=0
|
||
|
||
for i in $(seq 1335 1529); do
|
||
FILE="pubmed26n${i}.xml.gz"
|
||
URL="https://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/${FILE}"
|
||
|
||
log "--- [$((i-1334))/195] $FILE ---"
|
||
|
||
# Step 1: Download (single connection, reliable)
|
||
log " 下载中..."
|
||
if ! aria2c -x1 -s1 --retry-wait=3 --max-tries=3 \
|
||
"$URL" -d "$STAGING" -o "$FILE" \
|
||
>> "$LOG" 2>&1; then
|
||
log " ERROR: 下载失败，跳过"
|
||
SKIP=$((SKIP+1))
|
||
rm -f "$STAGING/$FILE"
|
||
continue
|
||
fi
|
||
|
||
# Step 2: Gzip integrity check
|
||
log " 验证完整性..."
|
||
if ! gzip -t "$STAGING/$FILE" 2>/dev/null; then
|
||
log " CORRUPT: 文件损坏，重试一次..."
|
||
rm -f "$STAGING/$FILE"
|
||
aria2c -x1 -s1 --retry-wait=3 --max-tries=3 \
|
||
"$URL" -d "$STAGING" -o "$FILE" \
|
||
>> "$LOG" 2>&1 || true
|
||
if ! gzip -t "$STAGING/$FILE" 2>/dev/null; then
|
||
log " FAILED: 重试后仍损坏，跳过"
|
||
SKIP=$((SKIP+1))
|
||
rm -f "$STAGING/$FILE"
|
||
continue
|
||
fi
|
||
fi
|
||
|
||
# Step 3: Prep workdir (only this file)
|
||
rm -f "$WORKDIR"/*.xml.gz
|
||
cp "$STAGING/$FILE" "$WORKDIR/"
|
||
|
||
# Step 4: Import via docker (--no-deps 避免每次重新启动 migrate)
|
||
log " 导入中..."
|
||
if docker compose -f "$COMPOSE_FILE" run --no-deps --rm \
|
||
-v /data/pubmed/updatefiles:/data/pubmed_baseline \
|
||
worker python /app/scripts/pubmed_baseline.py \
|
||
--dir /data/pubmed_baseline >> "$LOG" 2>&1; then
|
||
mv "$WORKDIR/$FILE" "$IMPORTED/"
|
||
GOOD=$((GOOD+1))
|
||
log " ✅ 成功: $FILE"
|
||
else
|
||
rm -f "$WORKDIR/$FILE"
|
||
FAIL=$((FAIL+1))
|
||
log " ❌ 导入失败: $FILE"
|
||
fi
|
||
|
||
# Clean staging
|
||
rm -f "$STAGING/$FILE"
|
||
done
|
||
|
||
log "=== 逐文件导入完成 ==="
|
||
log "结果: 成功=$GOOD 跳过=$SKIP 失败=$FAIL"
|
||
log "详见: $IMPORTED/"
|