#!/bin/bash
# this shell script check system disk IO delay for ccdb
 
source /OSM/script/os_load_base.sh
source /OSM/script/ccdb_get_ld_device.sh

G_MAX_SYSTEM_DISK_IO_DELAY_TIME=8
G_MAX_SYSTEM_DISK_SLOW_COUNT=3
G_SLOW_DEVICES_INFO_LIST="/OSM/data/Infrastructure/ccdb/tmp/slow_devices_info.list"
G_MDADM_UPDARE_SCRIPT="/OSM/script/ccdb_mdadm_update.sh"
G_DEVICE_NAME=
G_PRODUCT_MODEL=
G_ENGINE_ID=
G_LINKED_NUM=
G_NODE_ID=
G_SYSTEM_DISK_DEVICE=

# **************************************************************************** #
# Function Name: CCDB_MADM_UPDATE_MD200
# Description: 更新md200,根据type添加或剔除成员盘
# Parameter: 
# Return: 
# **************************************************************************** #
CCDB_MADM_UPDATE_MD200()
{
    L_DISK_ID=$1
    L_UPDARW_TYPE=$2
    L_DISK_NAME="ld${L_DISK_ID}a"
    
    sh ${G_MDADM_UPDARE_SCRIPT} $G_DEVICE_NAME $L_UPDARW_TYPE $L_DISK_ID $G_PRODUCT_MODEL $G_ENGINE_ID $G_LINKED_NUM $L_DISK_NAME $G_NODE_ID
    if [ $? -ne 0 ]; then
        OS_echo ""$0" check system disk IO delay, update mdadm fail, $L_UPDARW_TYPE."
        return 1
    fi 
    
    return 0
}

# **************************************************************************** #
# Function Name: CCDB_HANDLE_SLOW_DISK
# Description: IO时延慢检测结果处理
# Parameter: 
# Return: 
# **************************************************************************** #
CCDB_HANDLE_SLOW_DISK()
{
    L_DISK_ID=$1
    L_DISK_SLOW_INFO=
    L_SLOW_COUNT=

    L_DISK_SLOW_INFO=(`awk '/^'${L_DISK_ID}' /{print $1, $2}' ${G_SLOW_DEVICES_INFO_LIST}`)
    if [ -z "${L_DISK_SLOW_INFO}" ]; then
        let L_SLOW_COUNT=1
        echo "$L_DISK_ID $L_SLOW_COUNT" >> $G_SLOW_DEVICES_INFO_LIST
        OS_echo ""$0" CCDB add the disk to slow list, $L_DISK_ID, $L_SLOW_COUNT."
    else
        let L_SLOW_COUNT=${L_DISK_SLOW_INFO[1]}+1
        sed -i -r "s/^$L_DISK_ID .*$/$L_DISK_ID $L_SLOW_COUNT/" $G_SLOW_DEVICES_INFO_LIST
        if [ $L_SLOW_COUNT -ge $G_MAX_SYSTEM_DISK_SLOW_COUNT ]; then
            CCDB_MADM_UPDATE_MD200 $L_DISK_ID "DISK_OUT"
            OS_echo ""$0" CCDB remove disk from mdadm, $L_DISK_ID."
        fi
        OS_echo ""$0" CCDB update the disk to slow list, $L_DISK_ID, $L_SLOW_COUNT."
    fi
    
    return 0
}

# **************************************************************************** #
# Function Name: CCDB_HANDLE_NORMAL_DISK
# Description: ID时延正常检测结果处理
# Parameter: 
# Return: 
# **************************************************************************** #
CCDB_HANDLE_NORMAL_DISK()
{
    L_DISK_ID=$1
    L_DISK_SLOW_INFO=
    
    L_DISK_SLOW_INFO=(`awk '/^'${L_DISK_ID}' /{print $1, $2}' ${G_SLOW_DEVICES_INFO_LIST}`)
    if [ -n "${L_DISK_SLOW_INFO}" ]; then
        CCDB_MADM_UPDATE_MD200 $L_DISK_ID "DISK_IN"
        sed -i -r "/^$L_DISK_ID .*$/d" $G_SLOW_DEVICES_INFO_LIST
        OS_echo ""$0" CCDB add disk to mdadm, $L_DISK_ID."
    fi
    
    return 0
}

# **************************************************************************** #
# Function Name: CCDB_CHECK_SYSTEM_DISK_IO_DELAY
# Description: 检测系统盘IO时延
# Parameter: 
# Return: 
# **************************************************************************** #
CCDB_CHECK_SYSTEM_DISK_IO_DELAY() 
{
    L_DISK_ID=

    for disk_device in ${G_SYSTEM_DISK_DEVICE[@]}
    do
        OS_echo ""$0" check system disk device start, disk id: $disk_device"

        L_DISK_ID=`echo $disk_device | grep -o '[0-9]\+'`

        start_time=$(date +%s)
        io_delay_result=`dd if=$disk_device of=/dev/null bs=1024 count=1`
        if [ $? -ne 0 ]; then
            OS_echo ""$0" check system disk device fail, disk id: $disk_device."
        fi
        end_time=$(date +%s)

        let cost_time=$end_time-$start_time

        if [ $cost_time -ge $G_MAX_SYSTEM_DISK_IO_DELAY_TIME ]; then
            OS_echo ""$0" check system disk device is show, disk id: $disk_device, io_cost_time:$cost_time s."
            CCDB_HANDLE_SLOW_DISK ${L_DISK_ID}
        else
            CCDB_HANDLE_NORMAL_DISK ${L_DISK_ID}
        fi
        
        OS_echo ""$0" check system disk device end, disk id: $disk_device, io_cost_time:$cost_time s"
    done
    
    return 0
}


# --------------------------  main  ------------------------------------------ #
if [ $# -ne 5 ]; then
    OS_echo "usage:"$0" device_name product_model engine_id linked_num node_id"
    exit 1
fi

G_DEVICE_NAME=$1
G_PRODUCT_MODEL=$2
G_ENGINE_ID=$3
G_LINKED_NUM=$4
G_NODE_ID=$5

#获取系统盘列表
G_SYSTEM_DISK_DEVICE=$(CCDB_get_ready_nvRAM_disk $G_PRODUCT_MODEL $G_ENGINE_ID $G_LINKED_NUM $G_NODE_ID)
if [ -z "${G_SYSTEM_DISK_DEVICE}" ]; then
    OS_echo "usage:"$0" check system disk is not ready."
    exit 1
fi

touch  $G_SLOW_DEVICES_INFO_LIST

#检测系统盘IO时延
CCDB_CHECK_SYSTEM_DISK_IO_DELAY
if [ $? -ne 0 ]; then
    OS_echo ""$0" check system disk IO delay fail."
    exit 1
fi 

exit 0
