#!/bin/bash
LOG_FILE=/var/log/post_reboot.log

# 步骤1：检查并启动Harbor
check_harbor() {
  echo "[$(date)] 开始检查Harbor服务" >> $LOG_FILE
  cd /data/harbor || return 1
  if ! docker-compose ps | grep -q "Up"; then
    echo "[$(date)] 检测到Harbor未运行，正在启动..." >> $LOG_FILE
    docker-compose up -d 
  fi
  return 0
}

# 步骤2：挂载存储
mount_storage() {
  echo "[$(date)] 执行存储挂载" >> $LOG_FILE
  ansible all -i /opt/ai-deployer/clusters/ai/hosts -m shell -a "mount -a" >> $LOG_FILE 2>&1
  mount -a
}

# 步骤3：检查K8s Pod状态
check_pods() {
  echo "[$(date)] 检查Kubernetes Pod状态" >> $LOG_FILE
  kubectl get pods -A >> $LOG_FILE 2>&1
  return $(kubectl get pods -A | grep -c -E "Error|NodeAffinity")
}

# 步骤4：重启Gluster存储
restart_gluster() {

  #如何安装了glusterfs，则重启glusterfs
  if [[ -f /usr/sbin/gluster ]];then
    echo "[$(date)] 开始检查Gluster存储" >> $LOG_FILE
    for volume in $(gluster volume list); do
      echo "[$(date)] 处理卷: $volume" >> $LOG_FILE
      echo y | gluster volume stop $volume >> $LOG_FILE 2>&1
      gluster volume start $volume >> $LOG_FILE 2>&1
    done
  fi
}

# 步骤5：重启Web服务
restart_web() {
  echo "[$(date)] 重启Web服务" >> $LOG_FILE
  docker restart ai-mgr ai-user >> $LOG_FILE 2>&1
}
# 步骤6：清理异常Pod
clean_pods() {
  echo "[$(date)] 清理异常Pod" >> $LOG_FILE
  for pattern in  "CrashLoopBackOff"  "Error" "NodeAffinity"; do
    kubectl get pods -A | grep "$pattern" | awk '{print $1,$2}' | \
    xargs -L1 kubectl delete pods --force -n >> $LOG_FILE 2>&1
  done
}

main() {
  check_harbor
  restart_gluster
  mount_storage
  check_pods
  clean_pods
  restart_web
  echo "[$(date)] 所有恢复操作完成" >> $LOG_FILE
}
main