#!/bin/bash
#set -e 

function usage() {
    echo -e "\033[33mUsage:\033[0m operation.sh COMMAND"
    cat <<EOF
-------------------------------------------------------------------------------------
  add_node ai <ip>     to add a work node to the k8s cluster
  add_glusterfs-disk <storage-node-ip> <disk>     to add a new disk to glusterfs clusters
  del_node ai <ip>     to delete a work node from the k8s cluster
  del_namespace <namespace>     to delete a namespace
  update_web           to update the ai-web to new version
  upgrade_release <desired-version>     to upgrade the platform services when services changed
  update_edition       to update the platform edition to professional
  update_ai-deployer <git-username> <git-password>    to update the code of ai-deployer
  update_harbor_cert   to update harbor with domain and ssl 
  reboot_k8s_network   to reboot the k8s network

  reboot_fluentd       to reboot the fluentd
  update_gpu_label  <ip>  gpu_name<ie.A100>  to update the gpu 
  harbor_gc_clear  to clear the harbor garbage

EOF
}

function logger() {
  TIMESTAMP=$(date +'%Y-%m-%d %H:%M:%S')
  case "$1" in
    debug)
      echo -e "$TIMESTAMP \033[36mDEBUG\033[0m $2"
      ;;
    info)
      echo -e "$TIMESTAMP \033[32mINFO\033[0m $2"
      ;;
    warn)
      echo -e "$TIMESTAMP \033[33mWARN\033[0m $2"
      ;;
    error)
      echo -e "$TIMESTAMP \033[31mERROR\033[0m $2"
      ;;
    *)
      ;;
  esac
}

function add_node() {
  logger info "配置免密码登录"
  serverPassword=$(grep -r 'SERVER_PASSWORD' $BASE/ai.config)
  serverPassword=${serverPassword#*=}
  if [[ ! -f /root/.ssh/id_rsa.pub ]];then
    ssh-keygen -t rsa -b 2048 -N '' -f ~/.ssh/id_rsa
  fi
  auto_ssh_copy_id() {
    expect -c "set timeout -1
    spawn ssh-copy-id $1
    expect {
      *yes/no* {send -- yes\r;exp_continue;}
      *password:* {send -- $2\r;exp_continue;}
      eof        {exit 0;}
    }"
  }
  auto_ssh_copy_id $2 $serverPassword
  
  # 检查添加节点IP正则表达式
  [[ $2 =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { logger error "Invalid ip add:$2"; return 1; }
  
  NODE_IP="${@:2}"
  
  #如果新增节点是GPU主机,则安装驱动
  GPU_IPs=$(grep -E 'COMPUTE_NODE' $BASE/ai.config|grep GPU_TYPE|awk -F " " '{print $1}'|awk -F "=" '!($2 in IP){IP[$2];print $2}')
  if [[ $GPU_IPs =~ $NODE_IP ]];then
    logger info "为新增节点安装GPU驱动"
    ansible-playbook -i $BASE/ai/prepareinstall/hosts -e "NODE_TO_ADD=$NODE_IP" $BASE/ai/prepareinstall/addnode-gpudriverbefore.yml
    ansible-playbook -i $BASE/ai/prepareinstall/hosts -e "NODE_TO_ADD=$NODE_IP" $BASE/ai/prepareinstall/addnode-gpudriver.yml
  fi

  #添加节点至k8s集群
  logger info "正在添加节点至k8s集群"
  $BASE/ezctl add-node ai $NODE_IP
  
  #为新节点打标签 
  $BASE/bin/kubectl label node $NODE_IP machine=x86 app=explorer --overwrite
  if [[ $GPU_IPs =~ $NODE_IP ]];then
    $BASE/bin/kubectl label node $NODE_IP type=gpu --overwrite
    gpu_share=$(grep -E 'GPU_TYPE' $BASE/ai.config|grep -E $NODE_IP|awk -F " " '{print $2}'|awk -F "=" '!($2 in gpu_share){gpu_share[$2];print $2}')
    $BASE/bin/kubectl label node $NODE_IP gpushare=$gpu_share --overwrite
    gpu_type=$(grep -E 'GPU_TYPE' $BASE/ai.config|grep -E $NODE_IP|awk -F " " '{print $3}'|awk -F "=" '!($2 in gpu_type){gpu_type[$2];print $2}')
    $BASE/bin/kubectl label node $NODE_IP gpu=$gpu_type --overwrite
  fi
  #新增节点安装nvidia-container-runtime
  logger info "正在安装nvidia-container-runtime"
  ansible -i $BASE/clusters/ai/hosts $NODE_IP -m copy -a "src=$BASE/ai/ai-manifests/02-nvidia-container-runtime.sh dest=/opt/02-nvidia-container-runtime.sh"
  ansible -i $BASE/clusters/ai/hosts $NODE_IP -m shell -a "cd /opt && /bin/bash /opt/02-nvidia-container-runtime.sh"
  
  #如果出现新增节点cni0和mynet0同时存在的情况,需要删除mynet0并重启节点
  isMynet0Exist=$(ansible -i $BASE/clusters/ai/hosts $NODE_IP -m shell -a "ip route|grep mynet0")
  if [[ $isMynet0Exist =~ mynet0 ]];then
    logger info "正在重启主机"
    ansible -i $BASE/clusters/ai/hosts $NODE_IP -m shell -a "ip link del mynet0"
    ansible -i $BASE/clusters/ai/hosts $NODE_IP -m shell -a "reboot"
  fi
}

function add_glusterfs-disk() {
  STORAGE_TYPE=$(grep -r 'STORAGE_TYPE' $BASE/ai.config |grep -v '^#'|cut -d'=' -f2)
  if [[ $STORAGE_TYPE == glusterfs ]];then
    BASE=$(grep -E 'BASE' ./ai.config|cut -d'=' -f2)
    SSH_PORT=$(grep -E 'SSH_PORT' ./ai.config|cut -d'=' -f2)
    rm -rf $BASE/clusters/ai/glusterfs-8.6/hosts
    echo "[glusterfs]" >> $BASE/clusters/ai/glusterfs-8.6/hosts
    cat ai.config  | grep STORAGE_NODE_IP >> $BASE/clusters/ai/glusterfs-8.6/hosts
    sed -i  's/STO.*IP[0-9]=//g' $BASE/clusters/ai/glusterfs-8.6/hosts
    echo "$1    RAW_DISK=$2" >> $BASE/clusters/ai/glusterfs-8.6/hosts
    echo "[all:vars]" >> $BASE/clusters/ai/glusterfs-8.6/hosts
    sed -i "/\[all:vars/a base_dir=$BASE" $BASE/clusters/ai/glusterfs-8.6/hosts
    echo "ansible_ssh_port=$SSH_PORT" >> $BASE/clusters/ai/glusterfs-8.6/hosts

    ansible-playbook -i "$BASE/clusters/ai/glusterfs-8.6/hosts" "$BASE/playbooks/15.add-glusterfs-disk.yml"
  fi
}

function del_node() {
  #删除k8s node节点
  NODE_IP="${@:2}"

  logger info "正在删除k8s node节点"
  $BASE/ezctl del-node ai $NODE_IP
  
  #从配置文件中删除节点信息
  isNodeExist=$(sed -n '/^\[kube_node/,/^\[harbor/p' "$BASE/clusters/ai/hosts")
  if [[ $isNodeExist =~ $NODE_IP ]];then
    startLine=$(cat -n $BASE/clusters/ai/hosts|grep "\[kube_node\]"|awk '{print $1}')
    endLine=$(cat -n $BASE/clusters/ai/hosts|grep "\[harbor\]"|awk '{print $1}')
    nodeLine=$(cat -n $BASE/clusters/ai/hosts|head -n $endLine| tail -n +$startLine|grep $NODE_IP|awk '{print $1}')
    sed -i "$nodeLine d" $BASE/clusters/ai/hosts
    
    nodeLine2=$(cat -n $BASE/ai.config|grep COMPUTE_NODE_IP|grep $NODE_IP|awk '{print $1}')
    sed -i "$nodeLine2 d" $BASE/ai.config
  else
    logger info "节点已从集群删除"
  fi
}
#############增加常用操作#####
function delete_namespace()   {
  ./delete_namespace.sh $1
}

############################
function update_web() {
  logger debug "正在更新web应用"
  cd $BASE/ai/ai-manifests/ai-auto-deploy/
  ./install.sh
  logger info "web应用更新成功"
}
function harbor_gc_clear() {
  logger debug "正在清理harbor镜像"
  $BASE/harbornospace.sh
  logger info "harbor镜像清理成功"
}
function upgrade_3.1.4_to_3.2() {
  logger debug "正在升级底层平台服务"
  #更新kubeflow 1.3至1.4.1
  logger info "更新kubeflow 1.3至1.4.1"
  logger debug "正在卸载kubeflow 1.3"
  $BASE/bin/kubectl delete -R -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.3/
  $BASE/bin/kubectl delete ns kubeflow-user-example-com

  NAMESPACES=$($BASE/bin/kubectl get namespaces)
  
  logger debug "等待kubeflow 1.3卸载完成"
  until [[ ! $NAMESPACES =~ "kubeflow" ]] && [[ ! $NAMESPACES =~ "cert-manager" ]] && [[ ! $NAMESPACES =~ "knative-eventing" ]] && [[ ! $NAMESPACES =~ "knative-serving" ]] && [[ ! $NAMESPACES =~ "kubeflow-user-example-com" ]]
  do
    NAMESPACES=$($BASE/bin/kubectl get namespaces)
  done

  logger debug "正在安装kubeflow 1.4.1"
  ./ai/ai-manifests/ai-install.sh kubeflow
  
  #更新volcano 1.4至1.5.1-1
  logger info "更新volcano 1.4至1.5.1-1"
  logger debug "正在卸载volcano 1.4"
  $BASE/bin/kubectl delete -f $BASE/ai/ai-manifests/yaml/volcano-development.yaml
  
  NAMESPACES=$($BASE/bin/kubectl get namespaces)
  logger debug "等待volcano 1.4卸载完成"
  until [[ ! $NAMESPACES =~ "volcano-monitoring" ]] && [[ ! $NAMESPACES =~ "volcano-system" ]]
  do
    NAMESPACES=$($BASE/bin/kubectl get namespaces)
  done

  logger debug "正在安装volcano 1.5.1-1"
  ./ai/ai-manifests/ai-install.sh volcano

  #修改共享GPU插件
  edition=$(grep -r 'EDITION' $BASE/ai.config |cut -d'=' -f2)
  if [[ $edition == professional ]];then
    isFileExist=$(ansible -i $BASE/clusters/ai/hosts kube_master -m shell -a "ls /etc/kubernetes/")
    if [[ $isFileExist =~ "scheduler-policy-config.json" ]];then
      logger debug "正在卸载gpushare插件"
      ansible -i $BASE/clusters/ai/hosts kube_master -m shell -a "/bin/mv /etc/systemd/system/kube-scheduler.service.backup /etc/systemd/system/kube-scheduler.service;systemctl daemon-reload && systemctl restart kube-scheduler;rm /etc/kubernetes/scheduler-policy-config.json"
      #删除gpushare插件
      $BASE/bin/kubectl delete -f $BASE/ai/ai-manifests/yaml/gpushare-schd-extender.yaml
      #更新gpushare插件
      ./ai/ai-manifests/ai-install.sh gpushare
    fi
  fi
}

function upgrade_3.2_to_3.4() {
  logger debug "正在升级底层平台服务"
  #更新volcano 1.5至1.7
  logger info "更新volcano 1.5至1.7"
  logger debug "正在卸载volcano 1.7"
  $BASE/bin/kubectl delete -f $BASE/ai/ai-manifests/yaml/volcano-development.yaml

  NAMESPACES=$($BASE/bin/kubectl get namespaces)
  logger debug "等待volcano 1.5卸载完成"
  until [[ ! $NAMESPACES =~ "volcano-monitoring" ]] && [[ ! $NAMESPACES =~ "volcano-system" ]]
  do
    NAMESPACES=$($BASE/bin/kubectl get namespaces)
  done

  logger debug "正在安装volcano 1.7"
  ./ai/ai-manifests/ai-install.sh volcano

  logger debug "正在卸载gpushare插件"
  #删除gpushare插件
  $BASE/bin/kubectl delete -f $BASE/ai/ai-manifests/yaml/volcano-device-plugin.yaml
  #更新gpushare插件
  ./ai/ai-manifests/ai-install.sh gpushare

  #更新kubeflow 1.3至1.4.1
  logger info "更新kubeflow 1.4至1.5"
  logger debug "正在卸载kubeflow 1.4"
  $BASE/bin/kubectl delete -R -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/

  NAMESPACES=$($BASE/bin/kubectl get namespaces)

  logger debug "等待kubeflow 1.4卸载完成"
  until [[ ! $NAMESPACES =~ "kubeflow" ]] && [[ ! $NAMESPACES =~ "cert-manager" ]] && [[ ! $NAMESPACES =~ "knative-eventing" ]] && [[ ! $NAMESPACES =~ "knative-serving" ]]
  do
    NAMESPACES=$($BASE/bin/kubectl get namespaces)
  done

  logger debug "正在安装kubeflow 1.5"
  ./ai/ai-manifests/ai-install.sh kubeflow 
}

function upgrade_3.2_to_3.4() {
  logger info "卸载fluentd"
  logger info "更新fluentd"
  logger info "修改kube-apiserver.service"
}
function upgrade_release() {
  #获取当前主版本号
  CURRENT_VERSION=$(grep -r 'VERSION' $BASE/ai.config|awk -F"=" '{print $2}')
  CURRENT_MAIN_VERSION=$(echo $CURRENT_VERSION|awk -F'-' '{print $1}')
  #获取期望升级到的主版本号
  DESIRED_VERSION=$1
  DESIRED_MAIN_VERSION=$(echo $DESIRED_VERSION|awk -F'-' '{print $1}')
  
  #版本号大小比较方法
  function version_gt() { test "$(echo "$@" | tr " " "\n" | sort -V | head -n 1)" != "$1"; }
  function version_ge() { test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" == "$1"; }
  function version_lt() { test "$(echo "$@" | tr " " "\n" | sort -rV | head -n 1)" != "$1"; }
  function version_le() { test "$(echo "$@" | tr " " "\n" | sort -V | head -n 1)" == "$1"; }

  #期望升级的版本号必须大于当前版本号
  version_gt $DESIRED_MAIN_VERSION $CURRENT_MAIN_VERSION || { logger error "升级版本需要大于当前版本($CURRENT_MAIN_VERSION)"; exit 1; }
  
  #如果期望升级的版本号小于v3.2,则底层平台没有变更,仅更新web应用
  if version_lt $DESIRED_MAIN_VERSION v3.2;then
    #更新ai.config版本号
    sed -i "s#VERSION=$CURRENT_VERSION#VERSION=$DESIRED_VERSION#g" $BASE/ai.config
    update_web
  fi
  
  #如果期望升级的版本号大于等于v3.2,并且当前版本小于v3.2,则需要升级底层平台服务
  if version_ge $DESIRED_MAIN_VERSION v3.2 && version_lt $CURRENT_MAIN_VERSION v3.2;then
    upgrade_3.1.4_to_3.2

    #更新ai.config版本号
    sed -i "s#VERSION=$CURRENT_VERSION#VERSION=$DESIRED_VERSION#g" $BASE/ai.config
    update_web
  fi

  #如果期望升级的版本号大于等于v3.4,并且当前版本小于v3.4,则需要升级底层平台服务
  if version_ge $DESIRED_MAIN_VERSION v3.4 && version_lt $CURRENT_MAIN_VERSION v3.4;then
    upgrade_3.2_to_3.4

    #更新ai.config版本号
    sed -i "s#VERSION=$CURRENT_VERSION#VERSION=$DESIRED_VERSION#g" $BASE/ai.config
    update_web
  fi

  #如果当前版本号大于等于v3.2,则底层平台没有变更,仅更新web应用
  if version_ge $CURRENT_MAIN_VERSION v3.4;then
    #更新ai.config版本号
    sed -i "s#VERSION=$CURRENT_VERSION#VERSION=$DESIRED_VERSION#g" $BASE/ai.config
    update_web
  fi


  ####后续有新的版本变动,需要增加新的版本区间并修改上面的版本区间####
}

function update_edition() {
  #拉取最新镜像
  logger info "正在拉取最新镜像"
  ./update-images.sh
  
  edition=$(grep -r 'EDITION' $BASE/ai.config |cut -d'=' -f2)
  #专业版安装EFK
  if [[ $edition == professional ]];then
    efkIP=$(grep -r 'EFK_NODE_IP' $BASE/ai.config |cut -d'=' -f2|awk '{print $1}')
    # 检查efkIP正则表达式
    [[ $efkIP =~ ^(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})(\.(2(5[0-5]{1}|[0-4][0-9]{1})|[0-1]?[0-9]{1,2})){3}$ ]] || { logger error "Invalid ip efk:$efkIP"; return 1; }
    sed -i "s/__efk_ip__/$efkIP/g" "$BASE/clusters/ai/config.yml"
    sed -i 's/efk_install: "no"/efk_install: "yes"/g' "$BASE/clusters/ai/config.yml"
 
    logger info "正在安装EFK"
    ansible-playbook -i "$BASE/clusters/ai/hosts" -e "@$BASE/clusters/ai/config.yml" "$BASE/playbooks/07.cluster-addon.yml"
  fi
  
  #专业版安装GPU共享插件
  if [[ $edition == professional ]];then
    logger debug "安装GPU共享插件"
    /bin/cp -f $BASE/ai/ai-manifests/yaml/volcano-device-plugin.yaml.tmp $BASE/ai/ai-manifests/yaml/volcano-device-plugin.yaml
    sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/volcano-device-plugin.yaml
    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/yaml/volcano-device-plugin.yaml
  fi

  #专业版安装vGPU插件
  if [[ $edition == professional ]];then
    logger debug "05_安装vGPU插件"
    sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/vgpu/values.yaml
    $BASE/bin/helm install -n kube-system vgpu $BASE/ai/ai-manifests/yaml/vgpu/
  fi

  #专业版安装jwt组件
  if [[ $edition == professional ]];then
    logger debug "07_安装Kubeflow JWT"
    mysql_node_ip=$(grep -E 'MYSQL_NODE_IP' $BASE/ai.config | awk -F "=" '{print $2}')
    mysql_root_password=$(grep -E 'MYSQL_NODE_PASSWORD' $BASE/ai.config | awk -F "=" '{print $2}')

    /bin/cp -f $BASE/ai/ai-manifests/yaml/jwt.yaml.tmp $BASE/ai/ai-manifests/yaml/jwt.yaml
    sed -i "s/DB_URL_VALUE/$mysql_node_ip/g"  $BASE/ai/ai-manifests/yaml/jwt.yaml
    sed -i "s/DB_PASSWORD_VALUE/$mysql_root_password/g"  $BASE/ai/ai-manifests/yaml/jwt.yaml
    sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/jwt.yaml
    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/yaml/jwt.yaml
  fi

  #专业版安装kubeflow
  if [[ $edition == professional ]];then
    logger info "安装kebeflow 1.5"
    while ! kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.5/; do echo "Retrying to apply resources"; sleep 10; done
  fi
}

function update_ai-deployer() {
  logger debug "更新代码"
  git_username=$1
  git_password=$2
  /bin/cp -f $BASE/ai.config /root

  if [[ -d /root/ai-deployer ]];then
    rm -rf /root/ai-deployer && cd /root && git clone http://$git_username:$git_password@39.107.65.40/ai/ai-deployer.git
  else
    cd /root && git clone http://$git_username:$git_password@39.107.65.40/ai/ai-deployer.git
  fi

  cd /root/ai-deployer && /bin/cp -rp * $BASE
  /bin/cp -f /root/ai.config $BASE
}

function update_harbor_cert() {
  logger info "更新harbor 证书"
  harborExternalDomain=$(grep -r 'HARBOR_EXTERNAL_DOMAIN' $BASE/ai.config |cut -d'=' -f2|awk '{print $1}')
  sed -i '/HARBOR_EXTERNAL_DOMAIN/s/.*/HARBOR_EXTERNAL_DOMAIN: "'$harborExternalDomain'"/' clusters/ai/config.yml
  ansible-playbook -i "clusters/ai/hosts" -e "@clusters/ai/config.yml" "playbooks/16.update-harbor-cert.yml"
}

function reboot_k8s_network() {
  logger info "重启k8s网络"
  $BASE/bin/kubectl delete pods -n kube-system -l app=flannel --force --grace-period=0
}

function reboot_fluentd() {
  logger info "重启fluentd"
  $BASE/bin/kubectl delete pods -n kube-system -l k8s-app=fluentd-es  --force --grace-period=0
}

function update_gpu_label() {
  logger info "更新gpu label"
  $BASE/bin/kubectl label nodes $1 gpu=$2 --overwrite
}


function main() {
  BASE=$(grep -E 'BASE' ./ai.config|cut -d'=' -f2)
  DEFAULT_BASE=/etc/ai-deployer
  if [[ $BASE != $DEFAULT_BASE ]];then
    logger debug "正在配置工作目录"
    sed -i "s#$DEFAULT_BASE#$BASE#g" `grep -r $DEFAULT_BASE . -rl`
  fi
  #线上MINIO
  MINIO_URL=$(grep -E 'MINIO_URL' $BASE/ai.config|cut -d'=' -f2)

  #线上镜像仓库
  REGISTRY_URL=$(grep -E 'REGISTRY_URL' $BASE/ai.config|cut -d'=' -f2)
  REGISTRY_DOMAIN=${REGISTRY_URL#*//}
  #本地镜像仓库
  HARBOR_IP=$(grep -r 'HARBOR_DOMAIN' $BASE/ai.config |cut -d'=' -f2|awk '{print $1}')
  HARBOR_PORT=$(grep -r 'HARBOR_NODE_PORT' $BASE/ai.config |cut -d'=' -f2|awk '{print $1}')
  REGISTRY_LOCAL="$HARBOR_IP:$HARBOR_PORT"

  DEFAULT_REGISTRY_DOMAIN=registry.mopaas.com
  if [[ $REGISTRY_LOCAL != $DEFAULT_REGISTRY_DOMAIN ]];then
    logger debug "正在配置线上镜像仓库"
    sed -i "s#$DEFAULT_REGISTRY_DOMAIN#$REGISTRY_LOCAL#g" $BASE/install.sh
    sed -i "s#$DEFAULT_REGISTRY_DOMAIN#$REGISTRY_LOCAL#g" $BASE/operate.sh
    sed -i "s#$DEFAULT_REGISTRY_DOMAIN#$REGISTRY_LOCAL#g" $BASE/clean.sh
    sed -i "s#$DEFAULT_REGISTRY_DOMAIN#$REGISTRY_LOCAL#g" `grep -r $DEFAULT_REGISTRY_DOMAIN $BASE/ai/ai-manifests/kubeflow-manifests -rl`
  fi

  
  [ "$#" -gt 0 ] || { usage >&2; exit 2; }
 
  case "$1" in
      (add_node)
          [ "$#" -gt 2 ] || { usage >&2; exit 2; }
          add_node "${@:2}"
          ;;
      (add_glusterfs-disk)
          [ "$#" -gt 2 ] || { usage >&2; exit 2; }
          add_glusterfs-disk "${@:2}"
          ;;
      (del_node)
          [ "$#" -gt 2 ] || { usage >&2; exit 2; }
          del_node "${@:2}"
          ;;
       (del_namespace)
          [ "$#" -eq 2 ] || { usage >&2; exit 2; }
          del_namespace "$2"
          ;;
      (update_web)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          update_web
          ;;
      (upgrade_release)
          [ "$#" -eq 2 ] || { usage >&2; exit 2; }         
          upgrade_release "$2"
          ;;
      (update_edition)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          update_edition
          ;;
      (update_ai-deployer)
          [ "$#" -gt 2 ] || { usage >&2; exit 2; }
          update_ai-deployer "${@:2}"
          ;;
      (update_harbor_cert)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          update_harbor_cert
          ;;
      (reboot_k8s_network)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          reboot_k8s_network
          ;;
      (reboot_fluentd)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          reboot_fluentd
          ;;
      (harbor_gc_clear)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          harbor_gc_clear
          ;;
      (update_gpu_label)
          [ "$#" -gt 2 ] || { usage >&2; exit 2; }
          update_gpu_label "${@:2}"
          ;;
      (*)
          usage
          exit 0
          ;;
  esac
}
main "$@"