#!/bin/bash

declare -r fi_update_work="/opt/fiupdate"
declare fi_rollback_log=''
declare work_curPath=''
declare product_dir='/opt/fiupdate/NCE_info'
declare root_pwd=''
declare ossadm_pwd=''
declare admin_pwd=''
declare mainOms=''
declare fiweb=''

TMOUT=0
User_ID=`/usr/bin/id -u`
if [ $User_ID -ne 0 ] 
then
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] run user must be root"
    exit 1
fi

echo -n "es_input: os_pwd_root"
read -s os_pwd_root

echo ""

echo -n "es_input: os_pwd_ossadm"
read -s os_pwd_ossadm

echo ""

echo -n "es_input: fi_pwd_admin"
read -s fi_pwd_admin

root_pwd=${os_pwd_root}
ossadm_pwd=${os_pwd_ossadm}
admin_pwd=${fi_pwd_admin}

root_pwd=$(perl -e 'print quotemeta shift(@ARGV)' "${root_pwd}")
ossadm_pwd=$(perl -e 'print quotemeta shift(@ARGV)' "${ossadm_pwd}")
admin_pwd=$(perl -e 'print quotemeta shift(@ARGV)' "${admin_pwd}")

# 如果回滚OS后存在存量同步进程以及开机启动项，则kill进程，回退开机启动项
ifexcinventory=`ps -ef | grep 'analyzer_inventory_auto_start.sh' | grep -v 'grep'`
if [ -n "$ifexcinventory" ]
then
    ps -ef | grep 'analyzer_inventory_auto_start.sh' | grep -v 'grep' | awk '{print $2}' | xargs kill -9 >/dev/null 2>&1
fi
# invtory issue after need to delete
SysVerInfo=`cat /etc/os-release`
if [[ "$SysVerInfo" =~ "EulerOS" ]]; then
    SysVer="EulerOS"
elif [[ "$SysVerInfo" =~ "SLES" ]]; then
    SysVer="SUSE"    
fi
if [ "$SysVer" == "EulerOS" ]
then
    # 删除旧的自动化配置，防止回滚os补丁前未完全存量同步
    sed -i '/analyzer_inventory_auto_start.sh/d' /etc/rc.d/rc.local > /dev/null 2>&1
fi
if [ "$SysVer" == "SUSE" ]
then
    sed -i '/analyzer_inventory_auto_start.sh/d' /etc/rc.d/after.local > /dev/null 2>&1
fi
#delete analyzer_inventory_auto_start
if [ -f /tmp/upgrade_inventory/analyzer_inventory_auto_start.sh ]
then
    rm -fr /tmp/upgrade_inventory/analyzer_inventory_auto_start.sh > /dev/null 2>&1
fi

function get_cur_path()
{
    cd "$(dirname "${BASH_SOURCE-$0}")"
    work_curPath="${PWD}"
    fi_scriptName="$(basename "${BASH_SOURCE-$0}")"
    cd - >/dev/null
}

InitLogDir()
{
    TASKID=${1}
    TASK_ID_PATH="/opt/upgrade/easysuite_upgrade/taskmgr/${TASKID}"
    TASK_LOG_FILE="${TASK_ID_PATH}/task.log"
    TASK_PROGRESS_FILE="${TASK_ID_PATH}/task.progress"
    TASK_STATUS_FILE="${TASK_ID_PATH}/task.status"
    su - ossadm -c "mkdir -p ${TASK_ID_PATH};touch ${TASK_LOG_FILE};touch ${TASK_PROGRESS_FILE};touch ${TASK_STATUS_FILE}"
    
    echo "Progress=0" >${TASK_PROGRESS_FILE}
    echo "Status=start" >${TASK_STATUS_FILE}
    su - ossadm -c "id sopuser 2>/dev/null | grep -qw sopgroup && chgrp -R sopgroup ${TASK_ID_PATH}"
}

# fresh progress
fresh_result()
{
    echo "Progress=$1" >${TASK_PROGRESS_FILE}
    echo "Status=$2" >${TASK_STATUS_FILE}
}

# refresh fail tag
refresh_fail()
{
    fresh_result "100" "fail"
}

# refresh finish tag
fresh_finish()
{
    fresh_result "100" "finish"
}

check_finish()
{
    if [ "${NEW_TASK}"="FALSE" ]
    then
        grep -q "Rollback FusionInsight Successfully" ${fi_rollback_log}
        if [ $? -eq 0 ]
        then
            fresh_finish
            return 0
        fi
    fi
    return 1
}

logDef()
{
    local funcName="$1"; shift
    local logLevel="$1"; shift
    local lineNO="$1"; shift

    local logTime="$(date -d today +'%Y-%m-%d %H:%M:%S')"
    printf "[${logTime}] ${logLevel} $* [${g_scriptName}(${funcName}):${lineNO}]($$)\n" \
        >> "${fi_rollback_log}" 2>&1
}

log_error()
{
    logDef "${FUNCNAME[1]}" "ERROR" "$@"
}

log_info()
{
    logDef "${FUNCNAME[1]}" "INFO" "$@"
}

init_log()
{
    if [ ! -d "$fi_update_work/log/" ]
    then
        mkdir "$fi_update_work/log/"
    fi
    
    fi_rollback_log="$fi_update_work/log/rollback_work.log"
    [ ! -f "${fi_rollback_log}" ] && touch "${fi_rollback_log}"
    chmod 600 "${fi_rollback_log}"
    
}

trap 'onCtrlC' INT
function onCtrlC () {
    echo 'exit program'
    exit 1
}

function GetWorkPath()
{
    dir=`/usr/bin/dirname $0`
    ispoint=`/bin/echo $dir| /usr/bin/awk -F/ '{print $1}'`
    if [ "$ispoint" = "." ]
    then
        dir=`/bin/echo $dir|/usr/bin/sed 's/^.//'`
        curpath="`pwd`$dir"
    elif [ "$ispoint" = ".." ]
    then
        curpath="`pwd`/$dir"
    else
       curpath="$dir"
    fi

    return 0
}

function Unlock()
{
    cd /opt/fiupdate
    expect <<END
    spawn bash safeunlock.sh
    expect {
        "please input ossadm password:" {send "$ossadm_pwd\r"}
    }
    expect {
        "please input root password:" {send "$root_pwd\r"}
    }
    expect {
        "please input FI admin password:" {send "$admin_pwd\r"}
    }
    set timeout -1
    expect "*#"
    exit
END
}

# 返回值0 成功；1密码将过期；2密码过期；其它失败
function check_pwd_expired()
{
    webip=$1
    cd /opt/fiupdate/py_work
    script="check_pwd_util.pyc"
    if [ -f check_pwd_util.py ]
    then
        script="check_pwd_util.py"
    fi
    expect <<END
        spawn /opt/fiupdate/ODAESwiftDeployPlugin/ODAEDeployTool/rtsp/python/bin/python $script $webip
        set timeout 60
        exec sleep 10
        expect {
            "*assword*" {send "$admin_pwd\r"}
            timeout {send_user "No response!\n"
                    exit 10}
        }
        set timeout 180
        expect eof
        catch wait ret
        set exit_status [lindex \$ret 3]
        exit \$exit_status
END
}

function checkclusterstart()
{
    mainOms=$1
    fiweb=$2

    ping -c 3 ${fiweb} > /dev/null 2>&1
    pingfiweb=$?
    for ((j=1;j<=30;j++))
    do
        if [ $pingfiweb -ne 0 ]
        then
            echo "[`date '+%Y-%m-%d %H:%M:%S'`]---The OMS is starting. Please wait..."
            sleep 120
            ping -c 3 ${fiweb} > /dev/null 2>&1
            pingfiweb=$?
        else
            break
        fi
    done
    ping -c 3 ${fiweb} > /dev/null 2>&1
    if [ $? -ne 0 ]
    then
        echo "[`date '+%Y-%m-%d %H:%M:%S'`]---OMS startup times out,please check."
        return 1
    fi

    expect <<END
                spawn scp -p /opt/fiupdate/pre_config.sh root@${fiweb}:/opt
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$root_pwd\r"}
                }
                set timeout 120
                expect eof
END

    expect <<END
                spawn ssh -t root@${fiweb} "cd /opt/ ; bash pre_config.sh isupgradeos ; rm -rf /opt/pre_config.sh"
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$root_pwd\r"}
                }
                expect {
                    "*need to start cluster*" {exit 3}
                }
                exit 2
END
    upgradeos=$?
    
    #检查FI密码是否过期
    # 返回值0 成功；1密码将过期；2密码过期；其它失败
    check_pwd_expired "$fiweb"
    ret=$?
    if [ $ret -ne 0 ]
    then
        if [ $ret -eq 1 ]
        then
            echo "Warning, The FusionInsight password will expire, ignore and continue."
        elif [ $ret -eq 2 ]
        then
            echo "The password of the FusionInsight admin user has expired."
            echo "Change the password by referring to 'Changing the Password for the admin User of FusionInsight Manager' in the 'Administrator Guide'."
            echo "The password must be same as the input in the EasySuite 'Configure Solution and Parameters' page."
            return 1
        else
            echo "Error : Check The password of the FusionInsight failed."
            return 1
        fi
    else
        echo "Info: Check The password of the FusionInsight success."
    fi
    
    #检查FI集群是否已经启动
    if [ $upgradeos -eq 3 ]
    then
        for ((j=1;j<=240;j++))
        do
        sleep 60
        expect <<END
            spawn scp -p /opt/fiupdate/pre_config.sh root@${fiweb}:/opt
            expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$root_pwd\r"}
            }
            set timeout 120
            expect eof
END
    
        expect <<END
            spawn ssh -t root@${fiweb} "cd /opt/ ; bash pre_config.sh checkcluster ; rm -rf /opt/pre_config.sh"
            expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$root_pwd\r"}
            }
            expect {
                "*start cluster success*" {exit 0}
                "*start cluster failed*" {exit 1}
            }
            exit 2
END

        ret=$?
        if [ $ret -eq 2 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi check cluster is running..."
            continue
        fi
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi check cluster is finished, ret is $ret"
        return $ret
	    done
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi check cluster is timeout."
        return 1
    fi
    return 0
}

function locksafe()
{
    product_name=$1
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_name : all FI nodes need to restore safe mode "
    log_info "${LINENO}" " $product_name : all FI nodes need to restore safe mode "
    if [ ! -f "/opt/fiupdate/safelock.sh" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_name : restore safe mode script not exists"
        log_error "${LINENO}" " $product_name : restore safe mode script not exists "
        return 1
    fi
    
    cd /opt/fiupdate
    expect <<END
    spawn  bash safelock.sh
    expect "*assword:"
    send "$root_pwd\r"
    set timeout -1
    expect eof
END
   
    return 0
}

function PostRollback()
{
    product_name=$1
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_name:  start to do post rollback "
    log_info "${LINENO}" "$product_name: start to do post rollback "
    mainOms=`cat /opt/fiupdate/NCE_info/nodes_${product_name}.ini |grep mainOms= |sed 's/mainOms=//g'`

    expect <<END
            spawn ssh -t root@${mainOms} "cd /opt/fiupdate ; bash post_rollback_update_fi.sh"
            expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$root_pwd\r"}
            }
            expect "*assword:"
            send "$root_pwd\r"
            expect "*assword:"
            send "$admin_pwd\r"
            set timeout 1800
            expect eof
END

    sleep 5
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_name: start to get post rollback "
    log_info "${LINENO}" "$product_name: start to get post rollback "
    expect <<END
            spawn scp -p root@${mainOms}:/opt/fiupdate/log/post_rollback_update_fi.sh.log  /opt/fiupdate/log/
            expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$root_pwd\r"}
            }
            set timeout 180
            expect eof
END

    if [ ! -f "/opt/fiupdate/log/post_rollback_update_fi.sh.log" ]
    then
        sleep 5
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_name: start to get post rollback "
        log_info "${LINENO}" "$product_name: start to get post rollback "
        expect <<END
                spawn scp -p root@${mainOms}:/opt/fiupdate/log/post_rollback_update_fi.sh.log  /opt/fiupdate/log/
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$root_pwd\r"}
                }
                set timeout 180
                expect eof
END

    fi
	
    if [ ! -f "/opt/fiupdate/log/post_rollback_update_fi.sh.log" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_name: post rollback run Failed "
        log_error "${LINENO}" "$product_name: post rollback run Failed "
        return 1
    fi
    
    runresult=`cat /opt/fiupdate/log/post_rollback_update_fi.sh.log | grep 'rollback FusionInsight Successfully'`
    
    if [ ! -n "$runresult" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_name: post rollback run failed "
        log_error "${LINENO}" "$product_name: post rollback run Failed "
        return 1
    fi
    
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_name: post rollback run success "
    log_info "${LINENO}" "$product_name: post rollback run success "
    
    return 0
}

function scpfile()
{
    if [ ! -f "/opt/fiupdate/NCE_info/nodes.ini" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] FI nodes config file not exists,please check"
        log_error "${LINENO}" "platform nodelist json not exists,please check"
        return 1
    fi
    mainOms=`cat /opt/fiupdate/NCE_info/nodes.ini |grep mainOms= |sed 's/mainOms=//g'`
    expect <<END
        spawn scp -p -r /opt/fiupdate/ root@${mainOms}:/tmp
        expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$root_pwd\r"}
        }
        set timeout 600
        expect eof
END

    return 0
    
}

function prerollbackntp()
{
    product_ini=$1
    rm -rf iptmp.conf > /dev/null 2>&1
    cat $product_ini |grep finode= |sed 's/finode=//g' >>./iptmp.conf
    cat $product_ini |grep Analyzer_Calc_Storage | awk -F "=" {'print $2'} >>./iptmp.conf
    fi_nodes=`sort -u iptmp.conf`
    for fi_node in ${fi_nodes}
    do        
    expect <<END
                spawn scp -p /opt/fiupdate/pre_config.sh root@${fi_node}:/opt
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$root_pwd\r"}
                }
                set timeout -1
                expect eof
END
    
    expect <<END
                spawn ssh -t root@${fi_node} "cd /opt/ ; bash pre_config.sh ntp_rollback ; rm -rf /opt/pre_config.sh"
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$root_pwd\r"}
                }
                set timeout -1
                expect eof
END
        
    done
    rm -rf iptmp.conf > /dev/null 2>&1
}

runback()
{
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] start to do rollback work."
    log_info "${LINENO}" "start to do rollback work."
    
    product_name=$1
    if [ ! -f "/opt/fiupdate/NCE_info/nodes_${product_name}.ini" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_name: FI nodes config file not exists,please check."
        log_error "${LINENO}" "$product_name: platform nodelist json not exists,please check."
        return 1
    fi
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_name : start to do rollback work."
    log_info "${LINENO}" "$product_name : start to do rollback work."
    mainOms=`cat /opt/fiupdate/NCE_info/nodes_${product_name}.ini |grep mainOms= |sed 's/mainOms=//g'`
    
    
    expect <<END
        spawn ssh -t root@${mainOms} "cd /opt/fiupdate ; bash rollback_update_fi.sh"
        expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$root_pwd\r"}
        }
        expect "*assword:"
            send "$root_pwd\r"
        set timeout -1
        expect eof
END

    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_name : start to get rollbackwork result "
    log_info "${LINENO}" "$product_name : start to get rollbackwork result "
    expect <<END
        spawn scp -p root@${mainOms}:/opt/fiupdate/log/rollback_update_8021.log  /opt/fiupdate/log/
        expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$root_pwd\r"}
        }
        set timeout -1
        expect eof
END
    
    if [ ! -f "/opt/fiupdate/log/rollback_update_8021.log" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_name : backwork Run Failed "
        log_error "${LINENO}" "$product_name : backwork Run Failed "
        return 1
    fi
    
    runresult=`cat /opt/fiupdate/log/rollback_update_8021.log | grep 'run odae rollback script success'`
    
    if [ ! -n "$runresult" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_name : Backwork Run Failed "
        log_error "${LINENO}" "$product_name : Backwork Run Failed "
        return 1
    fi
    
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_name : Backwork Run Success "
    log_info "${LINENO}" "$product_name : Backwork Run Success "
    
    return 0
    
    
}

EasySuiteRollbackLog()
{
    expect <<END
        set timeout 30
        spawn ssh -t ossadm@${mainOms} "tail -1 /tmp/fiupdate/rollbackfiupdate.log"
        expect {
            "*assword*" {send "$ossadm_pwd\r"}
        }
        expect "*#"
        exit 2
END
}

isprocess()
{
    # check the rollback log.
    expect <<END
            spawn ssh -t root@${mainOms} "ps -ef | grep fi_rollback.sh | grep -v 'grep' > /opt/fiupdate/log/checkrollback.txt"
            expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$root_pwd\r"}
            }
            set timeout 60
            expect eof
END

    expect <<END
            spawn scp -p root@${mainOms}:/opt/fiupdate/log/checkrollback.txt /opt/fiupdate/log/
            expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$root_pwd\r"}
            }
            set timeout 60
            expect eof
END

    if [ ! -f "/opt/fiupdate/log/checkrollback.txt" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] checkrollback not exists. "
        log_info "${LINENO}" " checkrollback not exists. "
        return 1
    fi
    checkrollback=`cat /opt/fiupdate/log/checkrollback.txt | grep 'fi_rollback.sh'`
    if [ ! -n "$checkrollback" ]
    then
        log_info "${LINENO}" " check rollback process not exists.: try again. "
        return 1
    fi
    return 0

}

excRollback()
{
    expect <<END
        set timeout 30
        log_user 0
        spawn ssh root@${mainOms}
        expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "*assword*" {send "$root_pwd\r"}
        }
        expect {
            "*#*" {send "(echo '$admin_pwd') | $do_work\r"}
        }
        expect {
            "*#*" {send "exit\r"}
        }
END

}

function DoRollback()
{
    mainOms=$1
    fiweb=$2
    do_work="nohup bash /opt/fiupdate/tools/fi_rollback.sh $fiweb &"
    excRollback
    sleep 5
    isprocess
    excRollbackresult=$?
    for ((j=1;j<=30;j++))
    do
        if [ $excRollbackresult -ne 0 ]
        then
            log_info "${LINENO}" " ---check rollback process not exists, sleep 5s then retry..."
            Unlock
            unlockResult=`cat ${curpath}/log/unlock_safe.log | grep "script in stage unlock all FI nodes success"`
            if [ "x${unlockResult}" == "x" ]
            then
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] \033[41;30m Safe unlock is failed. \033[0m" >> $fi_update_work/log/rollback_work.log
                return 1
            else
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] \033[42;34m Safe unlock is success. \033[0m" >> $fi_update_work/log/rollback_work.log
            fi
            excRollback
            sleep 5
            isprocess >> $fi_update_work/log/rollback_work.log
            excRollbackresult=$?
        else
            break
        fi
    done

    for ((i=1;i<=1800;i++))
    do
        sleep 5
        EasySuiteRollbackLog >> ${TASK_LOG_FILE}
        expect <<END
            set timeout 30
            spawn ssh -t ossadm@${mainOms} "tail -1 /tmp/fiupdate/rollbackfiupdate.log"
            expect {
                "*assword*" {send "$ossadm_pwd\r"}
            }
            expect {
                "*FI Rollback success*" {exit 0}
                "*FI Rollback failed*" {exit 3}
            }
            exit 2
END
        ret=$?
        num=$i
        if [ $ret -eq 2 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi rollback is running..."
            if [ $num -eq 200 ];then
                fresh_result "33" "running"
            elif [ $num -eq 400 ];then
                fresh_result "46" "running"
            elif [ $num -eq 600 ];then
                fresh_result "59" "running"
            elif [ $num -eq 800 ];then
                fresh_result "72" "running"
            fi
            continue
        elif [ $ret -eq 3 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi rollback is finished, ret is $ret"
            return 1
        elif [ $ret -eq 1 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] the connection to the mainOms node is abnormal, waiting for 600s to retry the connection..."
            for ((i=1;i<=120;i++))
            do
            sleep 5
                echo -e "\n\n\n[`date '+%Y-%m-%d %H:%M:%S'`] the connection to the mainOms node is abnormal and is retrying to connect..."
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi rollback is running..."
            done
            for ((i=1;i<=1680;i++))
            do
            sleep 5
            expect <<END
                set timeout 30
                spawn ssh -t ossadm@${mainOms} "tail -1 /tmp/fiupdate/rollbackfiupdate.log"
                expect {
                    "*assword*" {send "$ossadm_pwd\r"}
                }
                expect {
                    "*FI Rollback success*" {exit 0}
                    "*FI Rollback failed*" {exit 1}
                }
                exit 2
END
            ret=$?
            if [ $ret -eq 2 ]
            then
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi rollback is running..."
                continue
            fi
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi rollback is finished, ret is $ret"
            return $ret
            done
        fi
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi rollback is finished, ret is $ret"
        return $ret
    done
    
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi rollback is timeout."
    return 1
}

function DoContinueRollback()
{
    mainOms=$1

    for ((i=1;i<=1800;i++))
    do
        sleep 5
        EasySuiteRollbackLog >> ${TASK_LOG_FILE}
        expect <<END
            set timeout 30
            spawn ssh -t ossadm@${mainOms} "tail -1 /tmp/fiupdate/rollbackfiupdate.log"
            expect {
                "*assword*" {send "$ossadm_pwd\r"}
            }
            expect {
                "*FI Rollback success*" {exit 0}
                "*FI Rollback failed*" {exit 3}
            }
            exit 2
END
        ret=$?
        if [ $ret -eq 2 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi rollback is running..."
            continue
        elif [ $ret -eq 3 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi rollback is finished, ret is $ret"
            return 1
        elif [ $ret -eq 1 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] the connection to the mainOms node is abnormal, waiting for 600s to retry the connection..."
            for ((i=1;i<=120;i++))
            do
            sleep 5
                echo -e "\n\n\n[`date '+%Y-%m-%d %H:%M:%S'`] the connection to the mainOms node is abnormal and is retrying to connect..."
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi rollback is running..."
            done
            for ((i=1;i<=1680;i++))
            do
            sleep 5
            expect <<END
                set timeout 30
                spawn ssh -t ossadm@${mainOms} "tail -1 /tmp/fiupdate/rollbackfiupdate.log"
                expect {
                    "*assword*" {send "$ossadm_pwd\r"}
                }
                expect {
                    "*FI Rollback success*" {exit 0}
                    "*FI Rollback failed*" {exit 1}
                }
                exit 2
END
            ret=$?
            if [ $ret -eq 2 ]
            then
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi rollback is running..."
                continue
            fi
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi rollback is finished, ret is $ret"
            return $ret
            done
        fi
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi rollback is finished, ret is $ret"
        return $ret
    done
    
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi rollback is timeout."
    return 1
}
function disable_main()
{
    fiweb=$1
    mainOms=$2
    expect <<END
                spawn scp -p /opt/fiupdate/pre_config.sh ossadm@${mainOms}:/tmp
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$ossadm_pwd\r"}
                }
                set timeout 120
                expect eof
END

    expect <<END
            spawn ssh -t ossadm@${mainOms} "su - root -c 'cd /tmp/ ; bash pre_config.sh disablemain ${fiweb} ${mainOms} ; rm -rf /tmp/pre_config.sh'"
            expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$ossadm_pwd\r"}
            }
            expect "*assword:"
            send "$root_pwd\r"

            set timeout 600
            expect eof
END

}

function checkagent_pidnum()
{
    product_ini=$1
    rm -rf iptmp.conf > /dev/null 2>&1
    cat $product_ini |grep finode= |sed 's/finode=//g' >>./iptmp.conf
    cat $product_ini |grep Analyzer_Calc_Storage | awk -F "=" {'print $2'} >>./iptmp.conf
    fi_nodes=`sort -u iptmp.conf`
    for fi_node in ${fi_nodes}
    do
    expect <<END
                spawn scp -p /opt/fiupdate/pre_config.sh root@${fi_node}:/opt
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$root_pwd\r"}
                }
                set timeout 120
                expect eof
END
    
    expect <<END
                spawn ssh -t root@${fi_node} "cd /opt/ ; bash pre_config.sh checkagentpid ; rm -rf /opt/pre_config.sh > /dev/null 2>&1"
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$root_pwd\r"}
                }
                set timeout 120
                expect eof
END

    done
    rm -rf iptmp.conf > /dev/null 2>&1
}

function check_runtime()
{
    product_ini=$1
    rm -rf iptmp.conf > /dev/null 2>&1
    cat $product_ini |grep finode= |sed 's/finode=//g' >>./iptmp.conf
    cat $product_ini |grep Analyzer_Calc_Storage | awk -F "=" {'print $2'} >>./iptmp.conf
    fi_nodes=`sort -u iptmp.conf`
    for fi_node in ${fi_nodes}
    do
    expect <<END
                spawn scp -p /opt/fiupdate/pre_config.sh root@${fi_node}:/opt
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$root_pwd\r"}
                }
                set timeout 120
                expect eof
END
    
    expect <<END
                spawn ssh -t root@${fi_node} "cd /opt/ ; bash pre_config.sh checkruntime ; rm -rf /opt/pre_config.sh > /dev/null 2>&1"
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$root_pwd\r"}
                }
                expect {
                    "*runtime is success*" {exit 0}
                    "*runtime is failed*" {exit 1}
                }
                exit 0
END

    result=$?
    if [ $result -ne 0 ]
    then
        log_error "${LINENO}" "check runtime is failed,please check."
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] /opt/huawei/Bigdata/common/runtime on FI node ${fi_node} is not a soft link, Please check." >> ${TASK_LOG_FILE}
        return 1
    fi
    done
    rm -rf iptmp.conf > /dev/null 2>&1
    return 0
}

function unset_parameters()
{
    unset root_pwd
    unset admin_pwd
}

clear_log()
{
    rm -rf /tmp/fiupdate
    rm -f /opt/fiupdate/log/task.log
    rm -f /opt/fiupdate/log/update_fi.sh.log
}


Main()
{
    GetWorkPath
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Start to rollback FusionInsight..."
    log_info "${LINENO}" "Start to rollback FusionInsight..."
    
    # Create the easysuite rollback log dir.
    TASKID=$1
    if [ "x${TASKID}" == "x" ]
    then
        TASKID="rollback_child_fi"
    fi
    InitLogDir ${TASKID}
    
    NEW_TASK="TRUE"
    [ -d ${TASK_ID_PATH} ] && NEW_TASK="FALSE"
    
    # 检查是否已经全部执行完成
    check_finish && return 0
    
    if [ ! -d "$product_dir" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] product conf dir not exists ,please run safeunlock.sh"
        log_error "${LINENO}" "product conf dir not exists ,please run safeunlock.sh"
        return 1
    fi

    Unlock
    unlockResult=`cat ${curpath}/log/unlock_safe.log | grep "script in stage unlock all FI nodes success"`
    if [ "x${unlockResult}" == "x" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] \033[41;30m Safe unlock is failed. \033[0m"
        log_error "${LINENO}" "Safe unlock is failed."
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Safe unlock is failed." >> ${TASK_LOG_FILE}
        return 1
    else
        fresh_result "5" "running"
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] \033[42;34m Safe unlock is success. \033[0m"
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Safe unlock is success." >> ${TASK_LOG_FILE}
    fi

    product_all=`find $product_dir/ -name "node*.ini" -type f`
    if [ ! -n "$product_all" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] the product config file not exists,please check"
        log_error "${LINENO}" " the product config file not exists,please check"
        return 1
    else
        log_info "${LINENO}" "precheck the product config file success"
    fi
    product_all=`find $product_dir/ -name "node*.ini" -type f`
    for product_ini in ${product_all}
    do
    {
        product_one=$(basename $product_ini .ini|sed 's/nodes_//g')
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Start to do $product_one product"
        log_info "${LINENO}" "Start to do $product_one product"
        mainOms=`cat $product_ini |grep mainOms= |sed 's/mainOms=//g'`
        fiweb=`cat $product_ini|grep fiweb | sed  's/fiweb=//g'`
        checkagent_pidnum $product_ini
        check_runtime $product_ini
        if [ $? -ne 0 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_one: check runtime is failed,please check." >> ${TASK_LOG_FILE}
            log_error "${LINENO}" "$product_one: check runtime is failed,please check."
            return 1
        fi
        break_out=`cat $product_ini|grep 'rollback=0'`
        if [ -n "${break_out}" ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] config show $product_one product break rollback stage"
            log_info "${LINENO}" " config show $product_one product break rollback stage"
            continue
        else
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] config show $product_one product not break rollback stage"
            log_info "${LINENO}" " config show $product_one product not break rollback stage"
        fi
        checkclusterstart $mainOms $fiweb >> ${TASK_LOG_FILE}
        if [ $? -ne 0 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_one: check cluster start failed,please check cluster status."
            log_error "${LINENO}" "$product_one: check cluster start failed,please check cluster status."
            return 1
        else
            fresh_result "15" "running"
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_one: check cluster start success."
            log_error "${LINENO}" "$product_one: check cluster start success."
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_one: check cluster start success." >> ${TASK_LOG_FILE}
        fi
        runback $product_one
        if [ $? -ne 0 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_one: backwork run odae rollback failed."
            log_error "${LINENO}" "$product_one: backwork run odae rollback failed."
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_one: backwork run odae rollback failed." >> ${TASK_LOG_FILE}
            return 1
        else
            fresh_result "20" "running"
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_one: backwork run odae rollback success."
            log_info "${LINENO}" "$product_one: backwork run odae rollback success."
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] $product_one: backwork run odae rollback success." >> ${TASK_LOG_FILE}
        fi    
        disable_main $fiweb $mainOms
        prerollbackntp ${product_all}

        expect <<END
                spawn ssh -t root@${mainOms} "ps -ef | grep fi_rollback.sh | grep -v 'grep' > /opt/fiupdate/log/checkcontinuerollback.txt"
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$root_pwd\r"}
                }
                set timeout 60
                expect eof
END

        expect <<END
                spawn scp -p root@${mainOms}:/opt/fiupdate/log/checkcontinuerollback.txt /opt/fiupdate/log/
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$root_pwd\r"}
                }
                set timeout 60
                expect eof
END

        if [ ! -f "/opt/fiupdate/log/checkcontinuerollback.txt" ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] checkcontinuerollback not exists. "
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] checkcontinuerollback not exists. " >> ${TASK_LOG_FILE}
            return 1
        fi
        checkifrollback=`cat /opt/fiupdate/log/update_fi_prepare.sh.log | grep 'CreateProject Successfully'`
        if [ "x${checkifrollback}" != "x" ];then
            checkcontinue=`cat /opt/fiupdate/log/checkcontinuerollback.txt | grep 'fi_rollback.sh'`
            if [ "x${checkcontinue}" == "x" ]
            then
                DoRollback $mainOms $fiweb
                if [ $? -ne 0 ]
                then
                    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Rollback FI failed with NCE product $product_one, mainOms is $mainOms"
                    log_error "${LINENO}" "Rollback FI failed with NCE product $product_one, mainOms is $mainOms"
                    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Rollback FI failed with NCE product $product_one, mainOms is $mainOms" >> ${TASK_LOG_FILE}
                    return 1
                else
                    fresh_result "85" "running"
                    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Rollback FI success with NCE product."
                    log_info "${LINENO}" "Rollback FI success with NCE product."
                    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Rollback FI success with NCE product." >> ${TASK_LOG_FILE}
                fi
            else
                DoContinueRollback $mainOms
                if [ $? -ne 0 ]
                then
                    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Rollback FI Continue failed with NCE product $product_one, mainOms is $mainOms"
                    log_error "${LINENO}" "Rollback FI Continue failed with NCE product $product_one, mainOms is $mainOms"
                    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Rollback FI Continue failed with NCE product $product_one, mainOms is $mainOms" >> ${TASK_LOG_FILE}
                    return 1
                fi
                fresh_result "85" "running"
            fi
        fi
        # 如果FI和NCE并行回滚，有可能FI回滚期间root又被禁止登录了，重新启用root登录来执行后置处理
        Unlock
        unlockResult=`cat ${curpath}/log/unlock_safe.log | grep "script in stage unlock all FI nodes success"`
        if [ "x${unlockResult}" == "x" ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] \033[41;30m Safe unlock is failed. \033[0m"
            log_error "${LINENO}" "Safe unlock is failed."
            return 1
        else
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] \033[42;34m Safe unlock is success. \033[0m"
        fi
        PostRollback $product_one

    }
    done
    # 如果FI和NCE并行升级，有可能FI升级期间root又被禁止登录了，重新启用root登录来加固
    Unlock
    unlockResult=`cat ${curpath}/log/unlock_safe.log | grep "script in stage unlock all FI nodes success"`
    if [ "x${unlockResult}" == "x" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] \033[41;30m Safe unlock is failed. \033[0m"
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Safe unlock is failed." >> $curpath/log/$log_file
        return 1
    else
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] \033[42;34m Safe unlock is success. \033[0m"
    fi
    for product_ini in ${product_all}
    do
    {
        product_one=$(basename $product_ini .ini|sed 's/nodes_//g')
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Start to do $product_one product"
        log_info "${LINENO}" "Start to do $product_one product"
        locksafe $product_one
        if [ $? -ne 0 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`]  $product_one: postwork lock all FI nodes failed"
            log_error "${LINENO}" "$product_one: postwork lock all FI nodes failed"
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`]  $product_one: postwork lock all FI nodes failed" >> ${TASK_LOG_FILE}
            return 1
        fi
    
    }
    done

    clear_log
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] \033[42;34m rollback check run success \033[0m"
    log_info "${LINENO}" "rollback check run success"    
    
}


SuperMain()
{
    Main $*
    mainresult=$?
    unset_parameters
    if [ $mainresult -eq 0 ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Rollback FusionInsight Successfully."
        fresh_finish
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Rollback FusionInsight Successfully." >> ${fi_rollback_log}
    else
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Rollback FusionInsight failed."
        fresh_result "100" "fail"
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Rollback FusionInsight failed." >> ${fi_rollback_log}
    fi
    return mainresult
}


get_cur_path
cd "${work_curPath}"

init_log || return $?

SuperMain $* &
superret=$?
exit $superret
