#!/bin/bash
#set -e

function logger() {
  TIMESTAMP=$(date +'%Y-%m-%d %H:%M:%S')
  case "$1" in
    debug)
      echo -e "$TIMESTAMP \033[36mDEBUG\033[0m $2"
      ;;
    info)
      echo -e "$TIMESTAMP \033[32mINFO\033[0m $2"
      ;;
    warn)
      echo -e "$TIMESTAMP \033[33mWARN\033[0m $2"
      ;;
    error)
      echo -e "$TIMESTAMP \033[31mERROR\033[0m $2"
      ;;
    *)
      ;;
  esac
}

function 01_label() {
  logger debug "01_为主机打标签"
  #重启fluentd
   $BASE/bin/kubectl delete pods -n kube-system  -l k8s-app=fluentd-es --force
  #为x86主机打标签
  x86_ips=$(grep -E 'MGR_NODE_IP|COMPUTE_NODE_IP' $BASE/ai.config|awk -F " " '{print $1}'|awk -F "=" '!($2 in IP){IP[$2];print $2}')
  for x86_ip in $x86_ips
  do
    $BASE/bin/kubectl label node $x86_ip machine=x86 app=explorer --overwrite
  done

  #为gpu主机打标签
  gpu_ips=$(grep -E 'GPU_TYPE' $BASE/ai.config|awk -F " " '{print $1}'|awk -F "=" '!($2 in IP){IP[$2];print $2}')
  for gpu_ip in $gpu_ips
  do
    $BASE/bin/kubectl label node $gpu_ip type=gpu --overwrite
    gpu_share=$(grep -E 'GPU_TYPE' $BASE/ai.config|grep -E $gpu_ip|awk -F " " '{print $2}'|awk -F "=" '!($2 in gpu_share){gpu_share[$2];print $2}')
    $BASE/bin/kubectl label node $gpu_ip gpushare=$gpu_share --overwrite
    gpu_type=$(grep -E 'GPU_TYPE' $BASE/ai.config|grep -E $gpu_ip|awk -F " " '{print $3}'|awk -F "=" '!($2 in gpu_type){gpu_type[$2];print $2}')
    $BASE/bin/kubectl label node $gpu_ip gpu=$gpu_type --overwrite
  done
  #为dcu主机打标签
  dcu_ips=$(grep -E 'DCU_TYPE' $BASE/ai.config|awk -F " " '{print $1}'|awk -F "=" '!($2 in IP){IP[$2];print $2}')
  for dcu_ip in $dcu_ips
  do
    $BASE/bin/kubectl label node $dcu_ip type=dcu --overwrite
    dcu_share=$(grep -E 'DCU_TYPE' $BASE/ai.config|grep -E $dcu_ip|awk -F " " '{print $2}'|awk -F "=" '!($2 in dcu_share){dcu_share[$2];print $2}')
    $BASE/bin/kubectl label node $dcu_ip gpushare=$dcu_share  hygon.com/dcu=true --overwrite
    dcu_type=$(grep -E 'DCU_TYPE' $BASE/ai.config|grep -E $dcu_ip|awk -F " " '{print $3}'|awk -F "=" '!($2 in dcu_type){dcu_type[$2];print $2}')
    $BASE/bin/kubectl label node $dcu_ip gpu=$dcu_type --overwrite
  done
}

#function 02_nvdia_container_runtime() {
#  logger debug "02_安装nvidia-container-runtime"
#  #在所有gpu主机上安装和配置nvidia-container-runtime
#  cat $BASE/ai.config | grep -E 'GPU_TYPE'|awk -F " " '{print $1}'|awk -F "=" '!($2 in IP){IP[$2];print $2}' > /tmp/gpu_nodes.tmp

#  ansible all -i /tmp/gpu_nodes.tmp -m copy -a "src=$BASE/ai/ai-manifests/02-nvidia-container-runtime.sh dest=/opt/02-nvidia-container-runtime.sh"
#  ansible all -i /tmp/gpu_nodes.tmp -m shell -a "cd /opt && /bin/bash /opt/02-nvidia-container-runtime.sh"
#  #ansible all -i /tmp/gpu_nodes.tmp -m shell -a "/usr/bin/nvidia-container-runtime -v"
#}

function 03_GPUshare_true() {
  #3、安装gpu共享 gpushare=true

#  edition=$(grep -r 'EDITION' $BASE/ai.config |cut -d'=' -f2)
#  if [[ $edition == professional ]];then
#    #将gpushare脚本分发到master节点
#    cat $BASE/ai.config | grep -E 'MGR_NODE_IP'|awk -F " " '{print $1}'|awk -F "=" '!($2 in IP){IP[$2];print $2}' > /tmp/master_nodes.tmp
#    sshPort=$(grep -r 'SSH_PORT' $BASE/ai.config |cut -d'=' -f2|awk '{print $1}')
#    echo "[all:vars]" >> /tmp/master_nodes.tmp
#    sed -i "/\[all:vars/a ansible_ssh_port=$sshPort" /tmp/master_nodes.tmp 

#    ansible all -i /tmp/master_nodes.tmp -m copy -a "src=$BASE/ai/ai-manifests/gpushare-scheduler-extender/scheduler-policy-config.json dest=/opt/scheduler-policy-config.json"
#    ansible all -i /tmp/master_nodes.tmp -m copy -a "src=$BASE/ai/ai-manifests/03-GPUshare-true.sh dest=/opt/03-GPUshare-true.sh"
#    ansible all -i /tmp/master_nodes.tmp -m shell -a "/bin/bash /opt/03-GPUshare-true.sh"
 
#    logger debug "03_安装GPU共享插件" 
#    /bin/cp -f $BASE/ai/ai-manifests/yaml/gpushare-schd-extender.yaml.tmp $BASE/ai/ai-manifests/yaml/gpushare-schd-extender.yaml
#    sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/gpushare-schd-extender.yaml
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/yaml/gpushare-schd-extender.yaml
#  fi

  #3、安装gpu共享 gpushare=true
  edition=$(grep -r 'EDITION' $BASE/ai.config |cut -d'=' -f2)
  if [[ $edition == professional ]];then
    logger debug "03_安装GPU共享插件"
    /bin/cp -f $BASE/ai/ai-manifests/yaml/volcano-device-plugin.yaml.tmp $BASE/ai/ai-manifests/yaml/volcano-device-plugin.yaml
    sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/volcano-device-plugin.yaml
    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/yaml/volcano-device-plugin.yaml
  fi
}

function 04_GPUshare_false() {
  #4、安装nvidia-docker独享插件 gpushare=false
  
  logger debug "04_安装GPU独享插件"
  /bin/cp -f $BASE/ai/ai-manifests/yaml/nvidia-device-plugin.yaml.tmp $BASE/ai/ai-manifests/yaml/nvidia-device-plugin.yaml
  sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/nvidia-device-plugin.yaml
  $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/yaml/nvidia-device-plugin.yaml
  $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/yaml/priority-classes.yaml
}

function 05_GPUshare_vGPU() {
  #5、安装vGPU插件 gpushare=vgpu
  #修改yaml文件中的参数--device-split-count=4值，表示单块gpu卡可以虚拟出几个vgpu卡

  edition=$(grep -r 'EDITION' $BASE/ai.config |cut -d'=' -f2)
  if [[ $edition == professional ]];then
    logger debug "05_安装vGPU插件"
    vgpuState=$($BASE/bin/helm ls -A|awk 'NR>1{print $1}')
    if [[ $vgpuState =~ vgpu ]];then
      logger info "vGPU插件已安装"
    else
      #/bin/cp -f $BASE/ai/ai-manifests/yaml/nvidia-device-plugin-vgpu.yaml.tmp $BASE/ai/ai-manifests/yaml/nvidia-device-plugin-vgpu.yaml
      #sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/nvidia-device-plugin-vgpu.yaml
      #$BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/yaml/nvidia-device-plugin-vgpu.yaml
      sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/vgpu/values.yaml
      $BASE/bin/helm install -n kube-system vgpu $BASE/ai/ai-manifests/yaml/vgpu/
    fi
  fi
}

function 06_mysql() {
  logger debug "06_安装ai-mysql和minio"
  mysql_node_ip=$(grep -E 'MYSQL_NODE_IP' $BASE/ai.config | awk -F "=" '{print $2}')
  mysql_root_password=$(grep -E 'MYSQL_NODE_PASSWORD' $BASE/ai.config | awk -F "=" '{print $2}')
  minio_user=$(grep -E 'MINIO_ACCESS_KEY' $BASE/ai.config | awk -F "=" '{print $2}')
  minio_password=$(grep -E 'MINIO_SECRET_KEY' $BASE/ai.config | awk -F "=" '{print $2}')
  ai_mysql_network=$(grep -E 'AI_MYSQL_NETWORK' $BASE/ai.config | awk -F "=" '{print $2}')
  #修改ai.cfg配置文件
  /bin/cp -f $BASE/ai/ai-manifests/ai-app/ai.cfg.tmp $BASE/ai/ai-manifests/ai-app/ai.cfg
  sed -i "s/MYSQL_NODE_IP/$mysql_node_ip/g" $BASE/ai/ai-manifests/ai-app/ai.cfg
  sed -i "s/MYSQL_PASSWORD/$mysql_root_password/g" $BASE/ai/ai-manifests/ai-app/ai.cfg
  sed -i "s/MINIO_USER/$minio_user/g" $BASE/ai/ai-manifests/ai-app/ai.cfg
  sed -i "s/MINIO_PASSWORD/$minio_password/g" $BASE/ai/ai-manifests/ai-app/ai.cfg
  /bin/cp -f $BASE/ai/ai-manifests/ai-app/ai.cfg $BASE/ai/ai-manifests/ai-app/common/config/minio/env
  /bin/cp -f $BASE/ai/ai-manifests/ai-app/ai.cfg $BASE/ai/ai-manifests/ai-app/common/config/mysql/env
  #修改docker-compose.yaml
  /bin/cp -f $BASE/ai/ai-manifests/ai-app/docker-compose.yml.tmp $BASE/ai/ai-manifests/ai-app/docker-compose.yml
  sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/ai-app/docker-compose.yml
  sed -i "s#AI_MYSQL_NETWORK#$ai_mysql_network#g" $BASE/ai/ai-manifests/ai-app/docker-compose.yml
  #执行安装
  cd $BASE/ai/ai-manifests/ai-app/ && $BASE/bin/docker-compose up -d
  logger debug "初始化minio"
  if [[ -d /opt/minio/ai-pic ]];then
    logger info "minio已初始化"
  else
    #生成Minio bucket
    sleep 15
    docker exec -it ai-minio  /bin/sh /init.sh
  fi

  #创建运维平台数据库
  if [[ ! -d /opt/db/ai_kop ]];then
    logger debug "正在创建运维平台数据库"
    sleep 15
    $BASE/bin/docker exec -it ai-mysql bash -c "mysql -uroot -p$mysql_root_password -e 'create DATABASE ai_kop CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci'"
  fi
}

function 07_jwt() {
  edition=$(grep -r 'EDITION' $BASE/ai.config |cut -d'=' -f2)
  if [[ $edition == professional ]];then
    logger debug "07_安装Kubeflow JWT"
    mysql_node_ip=$(grep -E 'MYSQL_NODE_IP' $BASE/ai.config | awk -F "=" '{print $2}')
    mysql_root_password=$(grep -E 'MYSQL_NODE_PASSWORD' $BASE/ai.config | awk -F "=" '{print $2}')

    /bin/cp -f $BASE/ai/ai-manifests/yaml/jwt.yaml.tmp $BASE/ai/ai-manifests/yaml/jwt.yaml
    sed -i "s/DB_URL_VALUE/$mysql_node_ip/g"  $BASE/ai/ai-manifests/yaml/jwt.yaml
    sed -i "s/DB_PASSWORD_VALUE/$mysql_root_password/g"  $BASE/ai/ai-manifests/yaml/jwt.yaml
    sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/jwt.yaml
    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/yaml/jwt.yaml
  fi
}

function 08_event() {
  #8、安装审计组件
  logger debug "08_安装审计组件"
  mysql_node_ip=$(grep -E 'MYSQL_NODE_IP' $BASE/ai.config | awk -F "=" '{print $2}')
  mysql_root_password=$(grep -E 'MYSQL_NODE_PASSWORD' $BASE/ai.config | awk -F "=" '{print $2}')
  /bin/cp -f $BASE/ai/ai-manifests/yaml/k8s-event.yaml.tmp $BASE/ai/ai-manifests/yaml/k8s-event.yaml
  sed -i "s/DB_URL/$mysql_node_ip/g" $BASE/ai/ai-manifests/yaml/k8s-event.yaml
  sed -i "s/DB_PASSWORD/$mysql_root_password/g" $BASE/ai/ai-manifests/yaml/k8s-event.yaml
  sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/k8s-event.yaml
  $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/yaml/k8s-event.yaml
}

function 09_volcano() {
  #9、安装volcano 1.4
#  logger debug "09_安装volcano"
#  /bin/cp -f $BASE/ai/ai-manifests/yaml/volcano-development-1.4.yaml.tmp $BASE/ai/ai-manifests/yaml/volcano-development.yaml
#  sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/volcano-development.yaml
#  $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/yaml/volcano-development.yaml

  #9、安装volcano 1.5.1-1
  logger debug "09_安装volcano"
  /bin/cp -f $BASE/ai/ai-manifests/yaml/volcano-development-1.7.yaml.tmp $BASE/ai/ai-manifests/yaml/volcano-development.yaml
  sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/volcano-development.yaml
  $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/yaml/volcano-development.yaml

  /bin/cp -f $BASE/ai/ai-manifests/yaml/volcano-development-1.7-uninstall.yaml.tmp $BASE/ai/ai-manifests/yaml/volcano-development-uninstall.yaml
  sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/volcano-development-uninstall.yaml
}

#function 10_kubeflow() {
  #10、安装kubeflow 1.3
#  edition=$(grep -r 'EDITION' $BASE/ai.config |cut -d'=' -f2)
#  if [[ $edition == professional ]];then
#    logger debug "10_安装kubeflow cert-manager"
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.3/001-cert-manager-cert-manager-kube-system-resources-base.yaml
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.3/002-cert-manager-cert-manager-crds-base.yaml
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.3/003-cert-manager-overlays-self-signed.yaml
  
#    logger debug "等待kubeflow cert-manager启动"
#    until [[ $podState == "Running Running Running" ]]
#    do
#      podState=`$BASE/bin/kubectl get pod -n cert-manager |awk 'NR>1{print $3}'`
#      podState=`echo $podState`
#      if [[ $podState == "Running Running Running" ]];then
#        sleep 30
#        logger info "kubeflow cert-manager容器正在运行中"
#      fi
#    done
 
#    logger debug "10_安装kebeflow 1.3"
#    if [[ -f /usr/bin/python ]];then
#      python $BASE/ai/ai-manifests/kubeflow-manifests/install.py
#    else
#      python3 $BASE/ai/ai-manifests/kubeflow-manifests/install.py
#    fi
#  fi   

  #10、安装kubeflow 1.4.1
#  edition=$(grep -r 'EDITION' $BASE/ai.config |cut -d'=' -f2)
#  if [[ $edition == professional ]];then
#    logger debug "10_安装kubeflow cert-manager"
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/001-cert-manager-cert-manager-base.yaml

#    logger debug "等待kubeflow cert-manager启动"
#    until [[ $podState == "Running Running Running" ]]
#    do
#      podState=`$BASE/bin/kubectl get pod -n cert-manager |awk 'NR>1{print $3}'`
#      podState=`echo $podState`
#      if [[ $podState == "Running Running Running" ]];then
#        sleep 30
#        logger info "kubeflow cert-manager容器正在运行中"
#      fi
#    done

#    logger debug "10_安装kebeflow 1.4.1"
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/002-cert-manager-kubeflow-issuer-base.yaml
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/003-istio-1-9-istio-crds-base.yaml
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/004-istio-1-9-istio-namespace-base.yaml
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/005-istio-1-9-istio-install-base.yaml

#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/008-knative-overlays-gateways.yaml
#    until [[ $isExist == "queue-proxy" ]]
#    do
#      isExist=`$BASE/bin/kubectl get image.caching.internal.knative.dev -n knative-serving|awk 'NR>1{print $1}'`
#      $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/008-knative-overlays-gateways.yaml
#      isExist=`$BASE/bin/kubectl get image.caching.internal.knative.dev -n knative-serving|awk 'NR>1{print $1}'`
#    done

#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/010-istio-1-9-cluster-local-gateway-base.yaml
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/011-kubeflow-namespace-kubeflow-namespace-base.yaml
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/012-kubeflow-roles-kubeflow-roles-base.yaml
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/013-istio-1-9-kubeflow-istio-resources-base.yaml
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/015-kfserving-overlays-kubeflow.yaml
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/016-katib-installs-katib-with-kubeflow.yaml
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/025-training-operator-overlays-kubeflow.yaml
#    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.4.1/026-mpi-job-overlays-kubeflow.yaml
#  fi
#}

function 10_kubeflow() {

  #10、安装kubeflow 1.5
  edition=$(grep -r 'EDITION' $BASE/ai.config |cut -d'=' -f2)
  if [[ $edition == professional ]];then
    logger info "安装kebeflow 1.5"
    sed -i "s/registry.mopaas.com/$REGISTRY_LOCAL/g" `grep -rl registry.mopaas.com $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.5`
    while ! kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.5/; do echo "Retrying to apply resources"; sleep 10; done
    sleep 10
    logger info "重新生成证书"
    $BASE/bin/kubectl delete jobs.batch -n kubeflow katib-cert-generator
    $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/kubeflow-manifests/manifest1.5/katib.yaml
  fi
}

function 11_backup() {
  isBackup=$(cat /etc/crontab)
  if [[ $isBackup =~ backup.sh ]];then
    logger info "已开启备份"
  else
    echo "0 0 * * * root /opt/ai-deployer/backup.sh" >> /etc/crontab
    service cron restart
  fi

}

function 12_delete_evicted_pod() {
  /bin/cp -f $BASE/ai/ai-manifests/yaml/delete-evicted-pods.yaml.tmp $BASE/ai/ai-manifests/yaml/delete-evicted-pods.yaml
  sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/delete-evicted-pods.yaml
  $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/yaml/delete-evicted-pods.yaml
}

function 13_nvidia_smi_process() {
  /bin/cp -f $BASE/ai/ai-manifests/yaml/nvidia-smi-process.yml.tmp $BASE/ai/ai-manifests/yaml/nvidia-smi-process.yml
  sed -i "s/REGISTRY_LOCAL/$REGISTRY_LOCAL/g" $BASE/ai/ai-manifests/yaml/nvidia-smi-process.yml
  $BASE/bin/kubectl apply -f $BASE/ai/ai-manifests/yaml/nvidia-smi-process.yml
}
  
function usage() {
    echo -e "\033[33mUsage:\033[0m ai-install.sh COMMAND"
    cat <<EOF
-------------------------------------------------------------------------------------
  all       to install all services for ai
  label     to label k8s nodes for ai
  gpushare  to install  gpushare device for ai
  mysql     to install mysql for ai
  jwt       to install jwt for ai
  event     to install event for ai
  volcano   to install volcano for ai
  kubeflow  to install kubeflow for ai
EOF
}

#命令执行成功@?为0；函数执行成功return 1
function main() {
  BASE=/opt/ai-deployer
  HARBOR_IP=$(grep -r 'HARBOR_DOMAIN' $BASE/ai.config |cut -d'=' -f2|awk '{print $1}')
  HARBOR_PORT=$(grep -r 'HARBOR_NODE_PORT' $BASE/ai.config |cut -d'=' -f2|awk '{print $1}')
  REGISTRY_LOCAL="$HARBOR_IP:$HARBOR_PORT"

  [ "$#" -gt 0 ] || { usage >&2; exit 2; }

  case "$1" in
      (all)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          01_label
          03_GPUshare_true
          04_GPUshare_false
          06_mysql
          07_jwt
          08_event
          09_volcano
          10_kubeflow
          05_GPUshare_vGPU
          11_backup
          12_delete_evicted_pod
	  13_nvidia_smi_process
          ;;
      (label)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          01_label
          ;;
      (gpushare)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          03_GPUshare_true
          04_GPUshare_false
          05_GPUshare_vGPU
          ;;
      (mysql)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          06_mysql
          ;;
      (jwt)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          07_jwt
          ;;
      (event)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          08_event
          ;;
      (volcano)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          09_volcano
          ;;
      (kubeflow)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          10_kubeflow
          ;;
      (backup)
          [ "$#" -eq 1 ] || { usage >&2; exit 2; }
          11_backup
          ;;
      (*)
          usage
          exit 0
          ;;
  esac
}

main "$@"