#!/usr/bin/env bash
set -Eeuo pipefail

BACKUP_ROOT=/home/admin/backups/sub2api
DATE_TAG=$(date +%Y%m%d)
TS=$(date +%Y%m%d_%H%M%S)
BACKUP_DIR="$BACKUP_ROOT/backup-date-$DATE_TAG"
PG_DIR="$BACKUP_DIR/postgres"
REDIS_DIR="$BACKUP_DIR/redis"
META_DIR="$BACKUP_DIR/meta"
LOG_FILE="$BACKUP_ROOT/backup.log"
CRON_LOG="$BACKUP_ROOT/cron.log"
KEY_FILE=/home/admin/.ssh/id_backup
TRASH_DIR=/home/admin/.trash
RETAIN_DAYS=5

POSTGRES_CONTAINER=postgres
POSTGRES_DB=sub2api
REDIS_CONTAINER=redis-base
REDIS_CONF=/home/admin/docker-projects/redis-base/redis.conf

TARGETS=(
  "tencent|22|ubuntu@111.229.63.121|/home/ubuntu/storage/sub2api"
  "mason1|6000|mason1@127.0.0.1|/home/ubuntu/storage/sub2api"
  "jp3|22|admin@31.58.223.35|/home/ubuntu/storage/sub2api"
)

FAILED_LINE="?"
PG_SNAPSHOT_ID=""
PG_SNAPSHOT_PID=""
REDIS_VERIFY_CONTAINER=""
REDIS_VERIFY_DIR=""

log() {
  printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG_FILE"
}

fail() {
  log "FATAL: $*"
  exit 1
}

move_to_trash() {
  local src="$1"
  local base
  local dest
  base=$(basename "$src")
  dest="$TRASH_DIR/${base}_$(date +%Y%m%d_%H%M%S)"

  mkdir -p "$TRASH_DIR" 2>/dev/null || true
  if mv "$src" "$dest" 2>/dev/null; then
    return 0
  fi

  if command -v sudo >/dev/null 2>&1; then
    sudo -n mkdir -p "$TRASH_DIR"
    sudo -n mv "$src" "$dest"
    return 0
  fi

  return 1
}

archive_dir_if_exists() {
  local src="$1"
  [ -d "$src" ] || return 0
  move_to_trash "$src"
}

cleanup_local_old_backups() {
  local dirs=()
  local idx
  mapfile -t dirs < <(find "$BACKUP_ROOT" -maxdepth 1 -mindepth 1 -type d -name 'backup-date-*' | sort)
  if [ "${#dirs[@]}" -le "$RETAIN_DAYS" ]; then
    log "本地保留策略检查完成：当前 ${#dirs[@]} 天，无需归档"
    return 0
  fi

  for ((idx=0; idx<${#dirs[@]}-RETAIN_DAYS; idx++)); do
    archive_dir_if_exists "${dirs[$idx]}"
    log "本地旧备份已归档: ${dirs[$idx]}"
  done
}

cleanup_remote_old_backups() {
  local name="$1"
  local port="$2"
  local userhost="$3"
  local dest_root="$4"

  ssh -i "$KEY_FILE" -o BatchMode=yes -o StrictHostKeyChecking=no -p "$port" "$userhost" \
    "DEST_ROOT='$dest_root' RETAIN_DAYS='$RETAIN_DAYS' bash -s" <<'REMOTE'
set -Eeuo pipefail
trash_dir="$DEST_ROOT/.trash"
mkdir -p "$trash_dir"
mapfile -t dirs < <(find "$DEST_ROOT" -maxdepth 1 -mindepth 1 -type d -name 'backup-date-*' | sort)
if [ "${#dirs[@]}" -le "$RETAIN_DAYS" ]; then
  exit 0
fi
for ((idx=0; idx<${#dirs[@]}-RETAIN_DAYS; idx++)); do
  dir="${dirs[$idx]}"
  mv "$dir" "$trash_dir/$(basename "$dir")_$(date +%Y%m%d_%H%M%S)"
done
REMOTE

  log "$name 旧备份保留策略已执行：仅保留最近 $RETAIN_DAYS 天"
}

wait_for_redis_dbsize() {
  local container="$1"
  local value=""
  for i in $(seq 1 60); do
    value=$(docker exec "$container" redis-cli --raw DBSIZE 2>/dev/null | tr -d '\r' || true)
    if printf '%s' "$value" | grep -Eq '^[0-9]+$'; then
      printf '%s\n' "$value"
      return 0
    fi
    sleep 1
  done
  return 1
}

finish_pg_snapshot() {
  if [ -n "$PG_SNAPSHOT_PID" ]; then
    {
      printf 'COMMIT;\n\\q\n' >&"${PG_SNAPSHOT[1]}"
    } 2>/dev/null || true
    wait "$PG_SNAPSHOT_PID" 2>/dev/null || true
    PG_SNAPSHOT_PID=""
    PG_SNAPSHOT_ID=""
  fi
}

cleanup() {
  local rc=$?
  finish_pg_snapshot
  if [ -n "$REDIS_VERIFY_CONTAINER" ]; then
    docker rm -f "$REDIS_VERIFY_CONTAINER" >/dev/null 2>&1 || true
    REDIS_VERIFY_CONTAINER=""
  fi
  if [ -n "$REDIS_VERIFY_DIR" ]; then
    archive_dir_if_exists "$REDIS_VERIFY_DIR"
    REDIS_VERIFY_DIR=""
  fi
  if [ "$rc" -ne 0 ]; then
    log "backup script aborted rc=$rc line=$FAILED_LINE"
  fi
  exit "$rc"
}

trap 'FAILED_LINE=$LINENO' ERR
trap cleanup EXIT

mkdir -p "$PG_DIR" "$REDIS_DIR" "$META_DIR"
touch "$LOG_FILE" "$CRON_LOG"

log "===== 开始备份（3-2-1: local + tencent + mason1 + jp3）====="
log "backup_dir=$BACKUP_DIR"
log "retention_days=$RETAIN_DAYS"

command -v docker >/dev/null 2>&1 || fail 'docker 不存在'
command -v redis-cli >/dev/null 2>&1 || fail 'redis-cli 不存在'
command -v rsync >/dev/null 2>&1 || fail 'rsync 不存在'
command -v ssh >/dev/null 2>&1 || fail 'ssh 不存在'
command -v pigz >/dev/null 2>&1 || fail 'pigz 不存在'
[ -f "$KEY_FILE" ] || fail "缺少备份私钥: $KEY_FILE"

docker ps --format '{{.Names}}' | grep -qx "$POSTGRES_CONTAINER" || fail "PostgreSQL 容器未运行: $POSTGRES_CONTAINER"
docker ps --format '{{.Names}}' | grep -qx "$REDIS_CONTAINER" || fail "Redis 容器未运行: $REDIS_CONTAINER"

PG_ENV_FILE=$(mktemp)
docker inspect "$POSTGRES_CONTAINER" --format '{{range .Config.Env}}{{println .}}{{end}}' > "$PG_ENV_FILE"
PGUSER=$(awk -F= '/^POSTGRES_USER=/{print $2}' "$PG_ENV_FILE")
PGPASS=$(awk -F= '/^POSTGRES_PASSWORD=/{print $2}' "$PG_ENV_FILE")
[ -n "$PGUSER" ] || fail '未获取到 POSTGRES_USER'
[ -n "$PGPASS" ] || fail '未获取到 POSTGRES_PASSWORD'

REDIS_PW=$(awk '/^requirepass /{print $2}' "$REDIS_CONF")
[ -n "$REDIS_PW" ] || fail '未获取到 Redis requirepass'
REDIS_IMAGE=$(docker inspect "$REDIS_CONTAINER" --format '{{.Config.Image}}')
[ -n "$REDIS_IMAGE" ] || fail '未获取到 Redis image'

log "正在备份 PostgreSQL: db=$POSTGRES_DB (consistent snapshot)"
coproc PG_SNAPSHOT {
  docker exec -i -e PGPASSWORD="$PGPASS" "$POSTGRES_CONTAINER" psql -X -qAt -U "$PGUSER" -d "$POSTGRES_DB"
}
printf 'BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY;\nSELECT pg_export_snapshot();\n' >&"${PG_SNAPSHOT[1]}"
IFS= read -r PG_SNAPSHOT_ID <&"${PG_SNAPSHOT[0]}" || fail '获取 PostgreSQL snapshot 失败'
[ -n "$PG_SNAPSHOT_ID" ] || fail 'PostgreSQL snapshot 为空'

PG_FILE="$PG_DIR/sub2api_${TS}.sql.gz"
docker exec -e PGPASSWORD="$PGPASS" "$POSTGRES_CONTAINER" pg_dump --snapshot="$PG_SNAPSHOT_ID" -U "$PGUSER" -d "$POSTGRES_DB" | pigz -9 > "$PG_FILE"
pigz -t "$PG_FILE"
log "PostgreSQL 备份成功 ($(du -h "$PG_FILE" | awk '{print $1}'))"

PG_COUNTS_RAW=$(docker exec -i -e PGPASSWORD="$PGPASS" "$POSTGRES_CONTAINER" psql -X -qAt -U "$PGUSER" -d "$POSTGRES_DB" <<SQL
BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY;
SET TRANSACTION SNAPSHOT '$PG_SNAPSHOT_ID';
select current_database() || '|' || current_user;
select 'users|' || count(*) from public.users;
select 'user_subscriptions|' || count(*) from public.user_subscriptions;
select 'usage_logs|' || count(*) from public.usage_logs;
select 'billing_usage_entries|' || count(*) from public.billing_usage_entries;
select 'settings|' || count(*) from public.settings;
COMMIT;
SQL
)
finish_pg_snapshot
PG_COUNTS=$(printf '%s\n' "$PG_COUNTS_RAW" | sed '/^BEGIN$/d;/^SET$/d;/^COMMIT$/d;/^$/d')

REDIS_FILE="$REDIS_DIR/dump_${TS}.rdb"
log '正在备份 Redis'
redis-cli -a "$REDIS_PW" --rdb "$REDIS_FILE" >/dev/null
[ -s "$REDIS_FILE" ] || fail 'Redis RDB 文件为空'
log "Redis 备份成功 ($(du -h "$REDIS_FILE" | awk '{print $1}'))"

log "正在验证 Redis RDB 可恢复性: image=$REDIS_IMAGE"
REDIS_VERIFY_DIR=$(mktemp -d "$BACKUP_ROOT/sub2api-redis-verify.XXXXXX")
cp "$REDIS_FILE" "$REDIS_VERIFY_DIR/dump.rdb"
REDIS_VERIFY_CONTAINER="redis-backup-verify-$TS"
docker rm -f "$REDIS_VERIFY_CONTAINER" >/dev/null 2>&1 || true
docker run -d --name "$REDIS_VERIFY_CONTAINER" -v "$REDIS_VERIFY_DIR:/data" "$REDIS_IMAGE" redis-server --dir /data --dbfilename dump.rdb --save '' --appendonly no >/dev/null
REDIS_BACKUP_DBSIZE=$(wait_for_redis_dbsize "$REDIS_VERIFY_CONTAINER") || fail 'Redis RDB 恢复后长时间未完成加载'
REDIS_BACKUP_KEYSPACE=$(docker exec "$REDIS_VERIFY_CONTAINER" redis-cli INFO keyspace | sed -n '1,20p')
log "Redis RDB 恢复校验成功: dbsize=$REDIS_BACKUP_DBSIZE"
docker rm -f "$REDIS_VERIFY_CONTAINER" >/dev/null 2>&1 || true
REDIS_VERIFY_CONTAINER=""
archive_dir_if_exists "$REDIS_VERIFY_DIR"
REDIS_VERIFY_DIR=""

MYSQL_STATUS_FILE="$META_DIR/mysql-status_${TS}.txt"
{
  echo 'mysql_backup=skipped'
  echo 'reason=no_live_mysql_on_80'
  echo 'note=之前 all-databases_*.sql.gz 实际是 mysqldump usage 文本，不能再伪造成功'
} > "$MYSQL_STATUS_FILE"
log 'MySQL 已跳过：80 当前没有真实可备份的本机 MySQL 实例，避免继续产出假备份'

SUMMARY_FILE="$META_DIR/summary_${TS}.txt"
{
  echo "timestamp=$TS"
  echo "backup_dir=$BACKUP_DIR"
  echo "postgres_file=$(basename "$PG_FILE")"
  echo "redis_file=$(basename "$REDIS_FILE")"
  echo "mysql_status=$(basename "$MYSQL_STATUS_FILE")"
  echo "postgres_db=$POSTGRES_DB"
  echo 'postgres_counts_begin'
  printf '%s\n' "$PG_COUNTS"
  echo 'postgres_counts_end'
  echo 'redis_backup_keyspace_begin'
  printf '%s\n' "$REDIS_BACKUP_KEYSPACE"
  echo "redis_dbsize|$REDIS_BACKUP_DBSIZE"
  echo 'redis_backup_keyspace_end'
} > "$SUMMARY_FILE"

sync_with_logs() {
  local name="$1"
  shift
  set +e
  "$@" 2>&1 | sed "s/^/[${name}] /" | tee -a "$LOG_FILE"
  local cmd_rc=${PIPESTATUS[0]}
  set -e
  return "$cmd_rc"
}

FAILED_TARGETS=()
for item in "${TARGETS[@]}"; do
  IFS='|' read -r NAME PORT USERHOST DEST_ROOT <<< "$item"
  DEST_DIR="$DEST_ROOT/backup-date-$DATE_TAG"
  log "开始同步到 $NAME: $USERHOST:$DEST_DIR"
  if ssh -i "$KEY_FILE" -o BatchMode=yes -o StrictHostKeyChecking=no -p "$PORT" "$USERHOST" "mkdir -p '$DEST_DIR' '$DEST_ROOT'" \
    && sync_with_logs "$NAME" rsync -az --partial --append-verify --delete --info=progress2,stats2 -e "ssh -i $KEY_FILE -o BatchMode=yes -o StrictHostKeyChecking=no -p $PORT" "$BACKUP_DIR/" "$USERHOST:$DEST_DIR/" \
    && sync_with_logs "$NAME-log" rsync -az --partial --append-verify -e "ssh -i $KEY_FILE -o BatchMode=yes -o StrictHostKeyChecking=no -p $PORT" "$LOG_FILE" "$CRON_LOG" "$USERHOST:$DEST_ROOT/"; then
    log "$NAME 同步成功"
    cleanup_remote_old_backups "$NAME" "$PORT" "$USERHOST" "$DEST_ROOT"
  else
    log "$NAME 同步失败"
    FAILED_TARGETS+=("$NAME")
  fi
done

if [ ${#FAILED_TARGETS[@]} -gt 0 ]; then
  fail "以下目标同步失败: ${FAILED_TARGETS[*]}"
fi

cleanup_local_old_backups

log '===== 备份完成 ====='
log "本地备份位置: $BACKUP_DIR"