#! /bin/bash

log_file="update_fi_distribute.sh.log"
task_file="task.log"

echo -n "es_input: os_pwd_root"
read -s os_pwd_root

echo ""

echo -n "es_input: os_pwd_ossadm"
read -s os_pwd_ossadm

echo ""

echo -n "es_input: fi_pwd_admin"
read -s fi_pwd_admin

GetWorkPath()
{
    dir=`/usr/bin/dirname $0`
    ispoint=`/bin/echo $dir| /usr/bin/awk -F/ '{print $1}'`
    if [ "$ispoint" = "." ]
    then
        dir=`/bin/echo $dir|/usr/bin/sed 's/^.//'`
        curpath="`pwd`$dir"
    elif [ "$ispoint" = ".." ]
    then
        curpath="`pwd`/$dir"
    else
       curpath="$dir"
    fi
    
    mkdir -p $curpath/log
    touch $curpath/log/$log_file
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Begin update FusionInsight..."
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Begin update FusionInsight..." >> $curpath/log/$log_file
    
    return 0
}

InitLogDir()
{
    TASKID=${1}
    TASK_ID_PATH="/opt/upgrade/easysuite_upgrade/taskmgr/${TASKID}"
    TASK_LOG_FILE="${TASK_ID_PATH}/task.log"
    TASK_PROGRESS_FILE="${TASK_ID_PATH}/task.progress"
    TASK_STATUS_FILE="${TASK_ID_PATH}/task.status"
    su - ossadm -c "mkdir -p ${TASK_ID_PATH};touch ${TASK_LOG_FILE};touch ${TASK_PROGRESS_FILE};touch ${TASK_STATUS_FILE}"
    su - ossadm -c "id sopuser 2>/dev/null | grep -qw sopgroup && chgrp -R sopgroup ${TASK_ID_PATH}"
}

# fresh progress
fresh_result()
{
    echo "Progress=$1" >${TASK_PROGRESS_FILE}
    echo "Status=$2" >${TASK_STATUS_FILE}
}

# refresh fail tag
refresh_fail()
{
    fresh_result "100" "fail"
}

# refresh finish tag
fresh_finish()
{
    fresh_result "100" "finish"
}

check_finish()
{
    if [ "${NEW_TASK}"="FALSE" ]
    then
        grep -q "FusionInsight Distribute Successfully" ${curpath}/log/${log_file}
        if [ $? -eq 0 ]
        then
            fresh_finish
            return 0
        fi
    fi
    return 1
}

InputParas()
{
    whouser=`whoami`
    if [ "$whouser" != "root" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] please run this script as user 'root' "
        exit 1
    fi
    
    root_pwd=${os_pwd_root}
    ossadm_pwd=${os_pwd_ossadm}
    admin_pwd=${fi_pwd_admin}
    
    ossadm_pwd=$(perl -e 'print quotemeta shift(@ARGV)' "${ossadm_pwd}")
    root_pwd=$(perl -e 'print quotemeta shift(@ARGV)' "${root_pwd}")
    admin_pwd=$(perl -e 'print quotemeta shift(@ARGV)' "${admin_pwd}")
}

Unlock()
{
    cd /opt/fiupdate
    expect <<END
    spawn bash safeunlock.sh
    expect {
        "please input ossadm password:" {send "$ossadm_pwd\r"}
    }
    expect {
        "please input root password:" {send "$root_pwd\r"}
    }
    expect {
        "please input FI admin password:" {send "$admin_pwd\r"}
    }
    set timeout 1200
    expect "*#"
    exit
END
}

PreUpdate()
{
    prepare_result=`cat $curpath/log/$task_file | grep "Prepare stage is success"`
    if [ "x${prepare_result}" != "x" ]
    then
        return 0
    fi
    cd /opt/fiupdate
    expect <<END
        spawn bash pre_update.sh
        expect {
            "please input root password:" {send "$root_pwd\r"}
        }
        expect {
            "please input admin password:" {send "$admin_pwd\r"}
        }
        set timeout -1
        expect eof

        catch wait ret
        set exit_status [lindex \$ret 3]
        exit \$exit_status
END
}

EasySuiteUpdateLog()
{
    expect <<END
        set timeout 30
        spawn ssh -t ossadm@${mainOms} "tail -1 /tmp/fiupdate/fiupdate_distribute.log"
        expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "*assword*" {send "$ossadm_pwd\r"}
        }
        expect "*#"
        exit
END
}

EasySuiteRetryUpdateLog()
{
    expect <<END
        set timeout 30
        spawn ssh -t ossadm@${mainOms} "tail -1 /tmp/fiupdate/retryfiupdate_distribute.log"
        expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "*assword*" {send "$ossadm_pwd\r"}
        }
        expect "*#"
        exit
END
}

isprocess()
{
    # check the upgrade log.
    expect <<END
            spawn ssh -t ossadm@${mainOms} "ps -ef | grep fi_update.sh | grep -v 'grep' > /opt/fiupdate/log/checkupgrade.txt"
            expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$ossadm_pwd\r"}
            }
            set timeout 60
            expect eof
END

    expect <<END
            spawn scp -p ossadm@${mainOms}:/opt/fiupdate/log/checkupgrade.txt /opt/fiupdate/log/
            expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$ossadm_pwd\r"}
            }
            set timeout 60
            expect eof
END

    if [ ! -f "/opt/fiupdate/log/checkupgrade.txt" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] checkupgrade not exists. "
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] checkupgrade not exists. " >> $curpath/log/$log_file
        return 1
    fi
    checkupgrade=`cat /opt/fiupdate/log/checkupgrade.txt | grep 'fi_update.sh distribute'`
    if [ ! -n "$checkupgrade" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] check upgrade process not exists.: try again. " >> $curpath/log/$log_file
        return 1
    fi
    return 0

}

excUpdate()
{
    expect <<END
        set timeout 30
        log_user 0
        spawn ssh root@${mainOms}
        expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "*assword*" {send "$root_pwd\r"}
        }
        expect {
            "*#*" {send "(echo '$ossadm_pwd' '$root_pwd' '$admin_pwd') | $do_work\r"}
        }
        expect {
            "*#*" {send "exit\r"}
        }
END

}

DoUpdate()
{
    mainOms=$1
    fiweb=$2
    do_work="nohup bash /opt/fiupdate/tools/fi_update.sh distribute $fiweb &"
    excUpdate
    # Tag that the upgrade has been performed.
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] update FI distribute execute, mainOms is $mainOms" >> $curpath/log/$task_file
    sleep 5
    isprocess >> $curpath/log/$log_file
    excUpdateresult=$?
    for ((j=1;j<=30;j++))
    do
        if [ $excUpdateresult -ne 0 ]
        then
            echo "[`date '+%Y-%m-%d %H:%M:%S'`]---check upgrade process not exists, sleep 5s then retry..." >> $curpath/log/$log_file
            Unlock
            unlockResult=`cat ${curpath}/log/unlock_safe.log | grep "script in stage unlock all FI nodes success"`
            if [ "x${unlockResult}" == "x" ]
            then
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Safe unlock is failed." >> $curpath/log/$log_file
                return 1
            else
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Safe unlock is success." >> $curpath/log/$log_file
            fi
            excUpdate
            sleep 5
            isprocess >> $curpath/log/$log_file
            excUpdateresult=$?
        else
            break
        fi
    done

    for ((i=1;i<=240;i++))
    do
        sleep 5
        EasySuiteUpdateLog >> ${TASK_LOG_FILE}
        expect <<END
            set timeout 30
            spawn ssh -t ossadm@${mainOms} "tail -1 /tmp/fiupdate/fiupdate_distribute.log"
            expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "*assword*" {send "$ossadm_pwd\r"}
            }
            expect {
                "*FI Distribute success*" {exit 0}
                "*FI Distribute failed*" {exit 3}
            }
            exit 2
END
        ret=$?
        if [ $ret -eq 2 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is running..."
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is running..." >> $curpath/log/$task_file
            continue
        elif [ $ret -eq 3 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret"
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret" >> $curpath/log/$task_file
            return 1
        elif [ $ret -eq 0 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret"
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret" >> $curpath/log/$task_file
            return 0
        elif [ $ret -eq 1 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] the connection to the mainOms node is abnormal, waiting for 600s to retry the connection..."
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] the connection to the mainOms node is abnormal, waiting for 600s to retry the connection..." >> $curpath/log/$task_file
            for ((i=1;i<=120;i++))
            do
            sleep 5
                echo -e "\n\n\n[`date '+%Y-%m-%d %H:%M:%S'`] the connection to the mainOms node is abnormal and is retrying to connect..."
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is running..."
            done
            for ((i=1;i<=240;i++))
            do
            sleep 5
            EasySuiteUpdateLog >> ${TASK_LOG_FILE}
            expect <<END
                set timeout 30
                spawn ssh -t ossadm@${mainOms} "tail -1 /tmp/fiupdate/fiupdate_distribute.log"
                expect {
                    "*yes/no*" {send "yes\r";exp_continue}
                    "*assword*" {send "$ossadm_pwd\r"}
                }
                expect {
                    "*FI Distribute success*" {exit 0}
                    "*FI Distribute failed*" {exit 3}
                }
                exit 2
END
            ret=$?
            if [ $ret -eq 0 ]
            then
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret"
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret" >> $curpath/log/$task_file
                return $ret
            fi
            if [ $ret -eq 3 ]
            then
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret"
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret" >> $curpath/log/$task_file
                return $ret
            fi
            continue
            done
        fi
        continue
    done
    
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is timeout."
    return 1
}

DoContinueUpdate()
{
    mainOms=$1

    for ((i=1;i<=240;i++))
    do
        sleep 5
        EasySuiteUpdateLog >> ${TASK_LOG_FILE}
        expect <<END
            set timeout 30
            spawn ssh -t ossadm@${mainOms} "tail -1 /tmp/fiupdate/fiupdate_distribute.log"
            expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "*assword*" {send "$ossadm_pwd\r"}
            }
            expect {
                "*FI Distribute success*" {exit 0}
                "*FI Distribute failed*" {exit 3}
            }
            exit 2
END
        ret=$?
        if [ $ret -eq 2 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is running..."
            continue
        elif [ $ret -eq 3 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret"
            return 1
        elif [ $ret -eq 1 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] the connection to the mainOms node is abnormal, waiting for 600s to retry the connection..."
            for ((i=1;i<=120;i++))
            do
            sleep 5
                echo -e "\n\n\n[`date '+%Y-%m-%d %H:%M:%S'`] the connection to the mainOms node is abnormal and is retrying to connect..."
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is running..."
            done
            for ((i=1;i<=240;i++))
            do
            sleep 5
            expect <<END
                set timeout 30
                spawn ssh -t ossadm@${mainOms} "tail -1 /tmp/fiupdate/fiupdate_distribute.log"
                expect {
                    "*yes/no*" {send "yes\r";exp_continue}
                    "*assword*" {send "$ossadm_pwd\r"}
                }
                expect {
                    "*FI Distribute success*" {exit 0}
                    "*FI Distribute failed*" {exit 1}
                }
                exit 2
END
            ret=$?
            if [ $ret -eq 2 ]
            then
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is running..."
                continue
            fi
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret"
            return $ret
            done
        fi
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret"
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret" >> $curpath/log/$task_file
        return $ret
    done
    
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is timeout."
    return 1
}

isretryprocess()
{
    # check the retryupgrade log.
    expect <<END
            spawn ssh -t ossadm@${mainOms} "ps -ef | grep fi_retryupdate.sh | grep -v 'grep' > /opt/fiupdate/log/checkretryupgrade.txt"
            expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$ossadm_pwd\r"}
            }
            set timeout 60
            expect eof
END

    expect <<END
            spawn scp -p ossadm@${mainOms}:/opt/fiupdate/log/checkretryupgrade.txt /opt/fiupdate/log/
            expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$ossadm_pwd\r"}
            }
            set timeout 60
            expect eof
END

    if [ ! -f "/opt/fiupdate/log/checkretryupgrade.txt" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] checkretryupgrade not exists. "
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] checkretryupgrade not exists. " >> $curpath/log/$log_file
        return 1
    fi
    checkretryupgrade=`cat /opt/fiupdate/log/checkretryupgrade.txt | grep 'fi_retryupdate.sh distribute'`
    if [ ! -n "$checkretryupgrade" ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] check retryupgrade process not exists.: try again. " >> $curpath/log/$log_file
        return 1
    fi
    return 0

}

excRetryUpdate()
{
    expect <<END
        set timeout 30
        log_user 0
        spawn ssh root@${mainOms}
        expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "*assword*" {send "$root_pwd\r"}
        }
        expect {
            "*#*" {send "(echo '$admin_pwd') | $do_work\r"}
        }
        expect {
            "*#*" {send "exit\r"}
        }
END

}

DoRetryUpdate()
{
    #If FI upgrade is successful, skip this step.
    upgrade_result=`cat $curpath/log/$task_file | grep "fi distribute is finished, ret is 0"`
    if [ "x${upgrade_result}" != "x" ]
    then
        return 0
    fi
    mainOms=$1
    do_work="nohup bash /opt/fiupdate/tools/fi_retryupdate.sh distribute &"
    excRetryUpdate
    sleep 5
    isretryprocess
    excRetryUpdateresult=$?
    for ((j=1;j<=30;j++))
    do
        if [ $excRetryUpdateresult -ne 0 ]
        then
            echo "[`date '+%Y-%m-%d %H:%M:%S'`]---check retryupgrade process not exists, sleep 5s then retry..." >> $curpath/log/$log_file
            Unlock
            unlockResult=`cat ${curpath}/log/unlock_safe.log | grep "script in stage unlock all FI nodes success"`
            if [ "x${unlockResult}" == "x" ]
            then
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Safe unlock is failed." >> $curpath/log/$log_file
                return 1
            else
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Safe unlock is success." >> $curpath/log/$log_file
            fi
            excRetryUpdate
            sleep 5
            isretryprocess >> $curpath/log/$log_file
            excRetryUpdateresult=$?
        else
            break
        fi
    done
  
    for ((i=1;i<=240;i++))
    do
        sleep 5
        EasySuiteRetryUpdateLog >> ${TASK_LOG_FILE}
        expect <<END
            set timeout 30
            spawn ssh -t ossadm@${mainOms} "tail -1 /tmp/fiupdate/retryfiupdate_distribute.log"
            expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "*assword*" {send "$ossadm_pwd\r"}
            }
            expect {
                "*FI Distribute success*" {exit 0}
                "*FI Distribute failed*" {exit 3}
            }
            exit 2
END
        ret=$?
        if [ $ret -eq 2 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is running..."
            continue
        elif [ $ret -eq 3 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret"
            return 1
        elif [ $ret -eq 1 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] the connection to the mainOms node is abnormal, waiting for 600s to retry the connection..."
            for ((i=1;i<=120;i++))
            do
            sleep 5
                echo -e "\n\n\n[`date '+%Y-%m-%d %H:%M:%S'`] the connection to the mainOms node is abnormal and is retrying to connect..."
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is running..."
            done
            for ((i=1;i<=240;i++))
            do
            sleep 5
            expect <<END
                set timeout 30
                spawn ssh -t ossadm@${mainOms} "tail -1 /tmp/fiupdate/retryfiupdate_distribute.log"
                expect {
                    "*yes/no*" {send "yes\r";exp_continue}
                    "*assword*" {send "$ossadm_pwd\r"}
                }
                expect {
                    "*FI Distribute success*" {exit 0}
                    "*FI Distribute failed*" {exit 1}
                }
                exit 2
END
            ret=$?
            if [ $ret -eq 2 ]
            then
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is running..."
                continue
            fi
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret"
            return $ret
            done
        fi
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret"
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is finished, ret is $ret" >> $curpath/log/$task_file
        return $ret
    done
    
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] fi distribute is timeout."
    return 1
}

checkIP()
{
    fiweb=$1
    fi_01=$2
    expect <<END
            spawn ssh -t ossadm@${fi_01} "ifconfig | grep '$fiweb'"
            expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$ossadm_pwd\r"}
            }
            expect {
                "*$fiweb*" {exit 0}
            }
            exit 2
END

    ret=$?
    return $ret
}

checkmaster()
{
    mainOms=$1
    fiweb=$2
    fi_01=$3
    backupOms=$4
    checkIP $fiweb $fi_01
    checkipret=$?
    if [ $checkipret -eq 2 ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] the node FI01 is not master, begin switchover."
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] the node FI01 is not master, begin switchover." >> $curpath/log/$task_file
        disable_backup $mainOms $fi_01 $backupOms
        checkIP $fiweb $fi_01
        checkipret=$?
        for ((j=1;j<=60;j++))
        do
            if [ $checkipret -ne 0 ]
            then
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] doing switchover. Please wait..." >> $curpath/log/$task_file
                sleep 5
                checkIP $fiweb $fi_01
                checkipret=$?
            else
                break
            fi
        done
        if [ $checkipret -eq 2 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] do switchover failed."
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] do switchover failed." >> $curpath/log/$task_file
            return 1
        elif [ $checkipret -eq 0 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] do switchover success."
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] do switchover success." >> $curpath/log/$task_file
            return 0
        fi
    else
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] the node FI01 is master, do not need switchover."
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] the node FI01 is master, do not need switchover." >> $curpath/log/$task_file
        return 0
    fi
}

function disable_backup()
{
    mainOms=$1
    fi_01=$2
    backupOms=$3
    expect <<END
                spawn scp -p /opt/fiupdate/pre_config.sh ossadm@${fi_01}:/tmp
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$ossadm_pwd\r"}
                }
                set timeout 1200
                expect eof
END

    expect <<END
            spawn ssh -t ossadm@${fi_01} "su - root -c 'cd /tmp/ ; bash pre_config.sh switchover ${backupOms} ${mainOms}'"
            expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$ossadm_pwd\r"}
            }
            expect "*assword:"
            send "$root_pwd\r"

            set timeout 1200
            expect eof
END

    sleep 15
    expect <<END
            spawn ssh -t ossadm@${fi_01} "su - root -c 'cd /tmp/ ; bash pre_config.sh disablebackup ${backupOms} ${mainOms} ; rm -rf /tmp/pre_config.sh'"
            expect {
            "*yes/no*" {send "yes\r";exp_continue}
            "assword:" {send "$ossadm_pwd\r"}
            }
            expect "*assword:"
            send "$root_pwd\r"

            set timeout 1200
            expect eof
END

}

Main()
{
    # Create the easysuite upgrade log dir.
    TASKID=$1
    if [ "x${TASKID}" == "x" ]
    then
        TASKID="upgrade_child_fi_distribute"
    fi
    InitLogDir ${TASKID}
    
    NEW_TASK="TRUE"
    [ -d ${TASK_ID_PATH} ] && NEW_TASK="FALSE"
    
    # 检查是否已经全部执行完成
    check_finish && return 0
    
    # Determine whether to retry. Only retry the DoUpdate step.
    
    InputParas
    Unlock
    unlockResult=`cat ${curpath}/log/unlock_safe.log | grep "script in stage unlock all FI nodes success"`
    if [ "x${unlockResult}" == "x" ]
    then
        sleep 90
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Safe unlock try again." >> $curpath/log/$log_file
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Safe unlock try again." >> ${TASK_LOG_FILE}
        Unlock
        unlockResult=`cat ${curpath}/log/unlock_safe.log | grep "script in stage unlock all FI nodes success"`
        if [ "x${unlockResult}" == "x" ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] \033[41;30m Safe unlock is failed. \033[0m"
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Safe unlock try again is failed." >> $curpath/log/$log_file
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Safe unlock try again is failed." >> ${TASK_LOG_FILE}
            return 1
        else
            fresh_result "10" "running"
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] \033[42;34m Safe unlock try again is success. \033[0m"
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Safe unlock try again is success." >> ${TASK_LOG_FILE}
        fi
    else
        fresh_result "10" "running"
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] \033[42;34m Safe unlock is success. \033[0m"
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Safe unlock is success." >> ${TASK_LOG_FILE}
    fi

    product_dir='/opt/fiupdate/NCE_info'
    product_all=`find $product_dir/ -name "nodes_*.ini" -type f`
    for product_ini in ${product_all}
    do
    {
        product_one=$(basename $product_ini .ini|sed 's/nodes_//g')
        mainOms=`cat $product_ini |grep mainOms= |sed 's/mainOms=//g'`
        fiweb=`cat $product_ini |grep fiweb | sed  's/fiweb=//g'`
        fi_01=`cat $product_ini |grep Analyzer_Calc_Storage_01= |sed 's/Analyzer_Calc_Storage_01=//g'`
        backupOms=`cat $product_ini |grep backupOms= |sed 's/backupOms=//g'`
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Start to check master node."
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Start to check master node." >> $curpath/log/$log_file
        checkmaster $mainOms $fiweb $fi_01 $backupOms
        if [ $? -ne 0 ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`]  $product_one: check master node failed."
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`]  $product_one: check master node failed." >> $curpath/log/$log_file
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`]  $product_one: check master node failed." >> ${TASK_LOG_FILE}
            return 1
        fi
    
    }
    done
    
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Begin to update FusionInsight in statge Distribute..." >> $curpath/log/$log_file
    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Begin to update FusionInsight in statge Distribute..." >> ${TASK_LOG_FILE}
    product_dir='/opt/fiupdate/NCE_info'
    product_all=`find $product_dir/ -name "nodes_*.ini" -type f`
    for product_ini in ${product_all}
    do
    {
        product_one=$(basename $product_ini .ini|sed 's/nodes_//g')
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] Start to do $product_one product"
        break_out=`cat $product_ini|grep 'prework=0'`
        if [ -n "${break_out}" ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] config show $product_one product break prework stage"
            continue
        else
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] config show $product_one product not break prework stage"
        fi
        
        mainOms=`cat $product_ini |grep mainOms= |sed 's/mainOms=//g'`
        fiweb=`cat $product_ini|grep fiweb | sed  's/fiweb=//g'`

        expect <<END
                spawn ssh -t ossadm@${mainOms} "ps -ef | grep update.sh | grep -v 'grep' > /opt/fiupdate/log/checkcontinueupgrade.txt"
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$ossadm_pwd\r"}
                }
                set timeout 60
                expect eof
END

        expect <<END
                spawn scp -p ossadm@${mainOms}:/opt/fiupdate/log/checkcontinueupgrade.txt /opt/fiupdate/log/
                expect {
                "*yes/no*" {send "yes\r";exp_continue}
                "assword:" {send "$ossadm_pwd\r"}
                }
                set timeout 60
                expect eof
END

        if [ ! -f "/opt/fiupdate/log/checkcontinueupgrade.txt" ]
        then
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] checkcontinueupgrade not exists. "
            echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] checkcontinueupgrade not exists. " >> $curpath/log/$task_file
            return 1
        fi
        checkcontinue=`cat /opt/fiupdate/log/checkcontinueupgrade.txt | grep 'fi_update.sh distribute'`
        checkretry=`cat /opt/fiupdate/log/checkcontinueupgrade.txt | grep 'fi_retryupdate.sh distribute'`
        if [ "x${checkcontinue}" == "x" -a "x${checkretry}" == "x" ]
        then
            DoUpdate_result=`cat $curpath/log/$task_file | grep "FI Distribute failed with NCE product"`
            UpdateExec_result=`cat $curpath/log/$task_file | grep "update FI distribute execute"`
            if [ "x${DoUpdate_result}" != "x" -o "x${UpdateExec_result}" != "x" ]
            then
                # If the FusionInsight update failed, the FusionInsight has been updated and the update failed. Therefore, try again.
                DoRetryUpdate $mainOms
                if [ $? -ne 0 ]
                then
                    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] update FI distribute retry failed with NCE product $product_one, mainOms is $mainOms"
                    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] update FI distribute retry failed with NCE product $product_one, mainOms is $mainOms" >> $curpath/log/$log_file
                    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] update FI distribute retry failed with NCE product $product_one, mainOms is $mainOms" >> $curpath/log/$task_file
                    return 1
                fi
            else
                # update fi not fail mean the fi is not to update.
                DoUpdate $mainOms $fiweb
                if [ $? -ne 0 ]
                then
                    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] FI Distribute failed with NCE product $product_one, mainOms is $mainOms"
                    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] FI Distribute failed with NCE product $product_one, mainOms is $mainOms" >> $curpath/log/$log_file
                    echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] FI Distribute failed with NCE product $product_one, mainOms is $mainOms" >> $curpath/log/$task_file
                    return 1
                fi
            fi
        else
            DoContinueUpdate $mainOms
            if [ $? -ne 0 ]
            then
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] update FI distribute Continue failed with NCE product $product_one, mainOms is $mainOms"
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] update FI distribute Continue failed with NCE product $product_one, mainOms is $mainOms" >> $curpath/log/$log_file
                echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] update FI distribute Continue failed with NCE product $product_one, mainOms is $mainOms" >> $curpath/log/$task_file
                return 1
            fi
        fi
        fresh_result "90" "running"
    }
    done

}

SuperMain()
{
    Main $*
    ret=$?
    if [ $ret -eq 0 ]
    then
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] FusionInsight Distribute Successfully."
        fresh_finish
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] FusionInsight Distribute Successfully." >> $curpath/log/$log_file
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] FusionInsight Distribute Successfully." >> ${TASK_LOG_FILE}
    else
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] FusionInsight Distribute failed."
        fresh_result "100" "fail"
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] FusionInsight Distribute failed." >> $curpath/log/$log_file
        echo -e "\n[`date '+%Y-%m-%d %H:%M:%S'`] FusionInsight Distribute failed." >> ${TASK_LOG_FILE}
    fi
    return $ret
}

GetWorkPath
SuperMain $* &
superret=$?
exit $superret
