--------------------------------------------------------------------------------
--                   Copyright, 2012-2013, Huawei Tech. Co., Ltd.
--                             All Rights Reserved
--------------------------------------------------------------------------------
-- Project Code    : OMM UPD V1.0
-- File name       : upd_flows.lua
-- Description     : Server侧升级流程模板
--------------------------------------------------------------------------------
-- History         :
--  1 Date         : 2012.12.20
--    Modification : 新增文件
--------------------------------------------------------------------------------

--执行状态表(该表内的所有数据都会持久化)
g_UpdState =
{
    Flow = "",              --当前正在执行的升级流程
    FlowId = 0,             --升级任务ID
    FlowState = 0,          --当前流程执行状态
                            --FLOW_STATE_RUNNING:正在执行
                            --FLOW_STATE_FINISH: 执行结束

    FlowUpdState = 0,      --当前的处于的升级状态

    AtomGroupIndex = 0,  --当前正在执行的原子组索引
    FailRbkAGIndex = 0,  --当前失败开始回退的原子组索引
    BatchNo = 0,         --当前分批的批次
    RbkBatchNo = 0,      --执行升级失败后的回退批次
    ParamList = {},      --参数列表

    NodeList = {},          --当前进行升级的所有节点列表
                            --{[ip]={ip(IP地址),tp(节点类型),st(升级状态),rt(当前原子组执行结果),bt(开始时间),et(结束时间),pr(进度)},{}}
    MasterList = {},          --当前进行升级的所有节点列表
                            --{[ip]={ip(IP地址),tp(节点类型),st(升级状态),rt(当前原子组执行结果),bt(开始时间),et(结束时间),pr(进度)},{}}
    CurNodeList = {},       --本批正在执行升级的节点列表
                            --{{ip(IP地址),tp(节点类型),rt(原子组执行结果)}, {}}
    SuccNodeList = {},      --本批执行成功的节点列表
                            --{{ip(IP地址),tp(节点类型),rt(原子组执行结果)}, {}}
    FailNodeList = {},      --本批执行失败的节点列表
                            --{{ip(IP地址),tp(节点类型),rt(原子组执行结果)}, {}}
    RbkNodeList = {},       --升级失败待回退的节点列表,每个节点包括{IP, 节点类型, 执行状态, AGIndex}
                            --{{ip(IP地址),tp(节点类型),rt(原子组执行结果),idx(原子组索引)}, {}}
    g_ClusterInfo = {},
    g_ClusInfoChange = 0,
    g_MasterIp = "",
    g_SwapMasterFalg = 0,
    g_MasterSlave = {},

    g_BackupFlowName = "",  --用于保存手动执行流程的流程名，执行自动流程(比如同步)后仍然可以查到手动执行流程的结果
    g_BackupFlowResult = 0, --用于保存手动执行流程的执行结果
    g_RebootNodeList = "",   --用于保存重启失败节点列表

    DistributePkgInfo = {},     --用于分发包

    g_ClusterType = 0,   ---1:表单控2:表示双控3:表示多控
    g_FlowStage = {stage = -1, batch = -1, agindex = -1},
    g_isCrossUpgrade = 0,
    g_MaxBatchNo  = 0,
    g_FinishAllBatch  = 0,
    g_GetNidType = 0, ---0:非6U设备使用NID只包含AB1:6U设备配置批次节点中使用NID包含0C 0D等
    g_CfgBatchNode = {},
    g_bHandleFault = 0, --是否处理fault状态下的升级流程
    g_abNormalRebootFlag = 0,

    AutoSuspendAtomGroupInfo = "",  --用于保存升级自动暂定信息，格式为：“2:2,3:5”，不同暂停点信息由逗号分隔，冒号分隔批次和暂停点对于的index，如2:5表示第二批升级系统前暂停
    AGPauseInfoArr = {},  --用于保存当前AG是否需要暂停
    g_AGPauseFalg = 0,
    g_CurNodeNumForPause = 0,  --标识当前已暂停的节点个数，用于暂停时保证当前批次所有节点都上报AG暂停状态
}

--================== UPD Framwork Const ========================================
local g_isPreStartFailed = 0; -- 通知SYS开始失败 需要再次通知SYS

local MODE_AMATEUR      = "0";
local MODE_PROFESSIONAL = "1";
local MODE_SHIELD_ALARM = "1"

local CON_TYPE_START    = "0";
local CON_TYPE_UPD      = "1";
local CON_TYPE_RETRY    = "2";
local CON_TYPE_ROLL     = "3";
local CON_TYPE_END      = "4";

local TRUE = 1
local FALSE = 0

local RET_OK = 0;                       --函数返回结果: 成功
local RET_ERR = -1;                     --函数返回结果: 失败


--FLOW状态
local FLOW_STATE_RUNNING = 0;           --流程执行状态: 正在执行
local FLOW_STATE_FINISH = 1;            --流程执行状态: 执行结束
local FLOW_STATE_UPD_SUSPEND = 2;
local FLOW_STATE_RBK_SUSPEND = 3;
local FLOW_STATE_END_SUSPEND = 4;
local FLOW_STATE_FAULT       = 5;

local MANAGER_FLOW_STATE_UPG = 0;       --Manager流程所处状态: 升级
local MANAGER_FLOW_STATE_UPG_RBK = 1;   --Manager流程所处状态: 升级失败回退

--调度状态
local FLOW_DISPATCH_STATE_NORMAL = 0;   --向节点分发任务状态: 正常
local FLOW_DISPATCH_STATE_FINISH = 1;   --向节点分发任务状态: 当前流程已经执行完成
local FLOW_DISPATCH_STATE_NO_FLOW = 2;  --向节点分发任务状态: 当前没有流程正在执行
local FLOW_DISPATCH_STATE_WRONG_IP = 3; --向节点分发任务状态: 节点IP地址错误
local FLOW_DISPATCH_STATE_UPD_SUSPEND = 4;
local FLOW_DISPATCH_STATE_RBK_SUSPEND = 5;
local FLOW_DISPATCH_STATE_END_SUSPEND = 6;
local FLOW_DISPATCH_STATE_UPD_PAUSE = 7;  --向节点分发任务状态: 升级不可回退暂停状态
local FLOW_DISPATCH_STATE_RBK_PAUSE = 8;  --向节点分发任务状态: 升级可回退暂停状态
local FLOW_DISPATCH_STATE_NEXT = 9;

--节点状态
local NODE_STATE_INIT = 0;              --节点升级状态: 未开始
local NODE_STATE_RUNNING = 1;           --节点升级状态: 执行中
local NODE_STATE_SUCC = 2;              --节点升级状态: 成功
local NODE_STATE_SUCC_RBK = 3;          --节点升级状态: 升级失败&回退成功(升级失败不需要回退的原子也配置成回退成功)
local NODE_STATE_FAIL = 4;              --节点升级状态: 回退失败
local NODE_STATE_CANCEL = 5;            --节点升级状态: 被自动取消
local NODE_STATE_TIMEOUT = 6;           --节点升级状态: 超时(由于工具不支持超时, 当作失败处理)

--AG状态
local ATOM_GROUP_EXEC_RET_INIT = -1;    --原子组执行结果: 未执行
local ATOM_GROUP_EXEC_RET_SUCC = 0;     --原子组执行结果: 执行成功
local ATOM_GROUP_EXEC_RET_FAIL = 2;     --原子组执行结果: 执行失败、或者 升级失败且回退失败
local ATOM_GROUP_EXEC_RET_SUCC_RBK = 3; --原子组执行结果: 回退成功
local ATOM_GROUP_EXEC_RET_TIMEOUT = 4;  --原子组执行结果: 执行超时
local ATOM_GROUP_EXEC_RET_PAUSE = 5;  --原子组执行结果: 暂停

--下一步动作
local EXEC_NEXT_AG      = 1;
local EXEC_AUTO_RBK     = 2;
local EXEC_END_FLOW     = 3;
local EXEC_UPD_SUSPEND  = 4;
local EXEC_RBK_SUSPEND  = 5;
local EXEC_END_SUSPEND  = 6; -- 一三阶段专家模式暂不用此状态

--离线升级失败率
local UPGRADE_FAIL_RATE = 50;


--是否切主
local MANAGER_SLAVE_CHANGE = 1;        --Manager流程所处状态: 回退
local MANAGER_SLAVE_NOT_CHANGE = 0;        --Manager流程所处状态: 回退


--设置FLOW和批次节点状态
local START_BATCH = 1;                  --节点状态更新事件类型: 开始处理一批节点
local END_BATCH = 2;                    --节点状态更新事件类型: 当前批次节点处理结束
local END_FLOW = 3;                     --节点状态更新事件类型: 当前流程处理结束
local TERM_FLOW = 4;                    --节点状态更新事件类型: 当前流程处理超时

--流程最后处理失败节点
local NO_NEED_REBOOT        = 0; --不需要重启
local REBOOT_CLUSTER        = 1; --重启集群
local REBOOT_FAIL_NODE      = 2; --重启失败节点

local BACKUP_FLOW_STATE_SUCC = 0;
local BACKUP_FLOW_STATE_FAIL = 1;
local BACKUP_FLOW_STATE_REBOOT = 2;
local BACKUP_FLOW_STATE_BUTT = 4;
local BATCH_BY_CONFIG          = 4; --根据配置文件获取批次
local BATCH_BY_CLUSTRINFO      = 5; --根据集群节点信息依次升最后升级主控
local BATCH_BY_DEFAULT         = 6; --630版本默认2批
local V3_6U_NID_TYPE           = 1; --6U多控包含0C 0D 1C 1D

local ABNORMAL_REBOOT_WHEN_HOTPATCH_ROLLBACK_OR_DELETE = 0;

-- 错误码 See <errno.h> for more details
local EPERM   = 1;  -- Operation not permitted
local EBADF   = 9;  -- Bad file descriptor

local STAGE_NOT_ENABLE  = -1; -- disable
local STAGE_INIT_FLOW   = 0;  -- init
local STAGE_FIRST_FLOW  = 1;  -- pre
local STAGE_SECOND_FLOW = 2;  -- online or offline
local STAGE_THIRD_FLOW  = 3;  -- post

local STAGE_INIT_BATCH   = 0;
local STAGE_FIRST_BATCH  = 1;
local STAGE_SECONE_BATCH = 2;
local FIRST_STAGE_MAX_BATCH    = 2;
local THIRD_STAGE_MAX_BATCH    = 3;
local REPAIR_PKG_SPCVERSION  = "";
local REPAIR_ROLL_SPCVERSION = "";


--================== UPD Framwork Const ========================================


--================== UPD Framwork API ==========================================
--------------------------------------------------------------------------------
-- Func Name       : LogText(level, message)
-- Description     : 日志打印函数
-- Caution         : 由升级框架注册给Lua使用
-- Input           : level: 日志级别, LL_DEBUG, LL_INFO, LL_WARN, LL_ERROR, LL_NOTICE
--                   message: 待打印的消息
-- Output          : 无
-- Return          : 无
--------------------------------------------------------------------------------

--------------------------------------------------------------------------------
-- Func Name       : ExecAtomGroup(flow_id, atom_group, node_ip, time_out, param_list)
-- Description     : 通知节点执行指定的AtomGroup
-- Caution         : 由升级框架注册给Lua使用
-- Input           : atom_group: string 待执行的AtomGroup名称
--                   node_ip: string 节点的IP地址
--                   timeout: int AtomGroup的超时时间,建议取Atom的最大超时时间
--                   param_list: string 从Server侧传递到Agent侧的参数列表
-- Output          : 无
-- Return          : 消息发送结果, 0 -- 成功 -1 -- 失败
--------------------------------------------------------------------------------

--------------------------------------------------------------------------------
-- Func Name       : System(cmd, timeout)
-- Description     : 执行系统调用
-- Caution         : 由升级框架注册给Lua使用
-- Input           : cmd: 命令行字符串
--                   timeout: 命令行执行超时时间，单位为分钟
-- Output          : 无
-- Return          : 0-127 -- 子进程的返回值 128 -- 子进程异常结束
--                   129 -- 子进程超时 130 -- 内部错误
--------------------------------------------------------------------------------

--------------------------------------------------------------------------------
-- Func Name       : SyncState(file_name)
-- Description     : 同步升级状态文件
-- Caution         : 由升级框架注册给Lua使用
-- Input           : file_name: 待同步的升级状态文件名，包含全路径
-- Output          : 无
-- Return          : 无
--------------------------------------------------------------------------------

--------------------------------------------------------------------------------
-- Func Name       : ReportAtomExecResult(node_ip, atom_group, atom, result, progress)
-- Description     : 上报原子执行结果
-- Caution         : 由升级框架注册给Lua使用
-- Input           : node_ip: 本节点IP地址    atom_group: AtomGroup名称
--                   atom: Atom名称           result: Atom执行结果
--                   progress: 当前AtomGroup执行的进度
-- Output          : 无
-- Return          : 返回上报消息发送是否成功 0 -- 成功 1 -- 失败
--------------------------------------------------------------------------------

--------------------------------------------------------------------------------
-- Func Name       : GetBackupFlowInfo
-- Description     : 获取保存的手动执行流程的执行结果
-- Caution         : 由OMM维护, 请勿修改
-- Input           : 无
-- Output          : 无
-- Return          : 返回流程名称与执行结果
--------------------------------------------------------------------------------
function GetBackupFlowInfo()
    return g_UpdState.g_BackupFlowName, g_UpdState.g_BackupFlowResult;
end

--------------------------------------------------------------------------------
-- Func Name       : file_exists
-- Description     : 判断指定的文件是否存在
--------------------------------------------------------------------------------
function file_exists(path)
    local file = io.open(path, "rb")
    if file then
        file:close()
        return true;
    end
    return nil;
end
--------------------------------------------------------------------------------
-- Func Name       : GetBackupRebootNodeInfo
-- Description     : 获取保存的重启失败节点信息
-- Caution         : 由OMM维护, 请勿修改
-- Input           : 无
-- Output          : 无
-- Return          : 返回重启失败节点信息
--------------------------------------------------------------------------------
function GetBackupRebootNodeInfo()
    return g_UpdState.g_RebootNodeList;
end

function UpdateRebootNodeList(Nid)
    local nid = Nid .. ","  ;
    LogText(LL_NOTICE, "The normal node is" .. nid);
    g_UpdState.g_RebootNodeList = string.gsub(g_UpdState.g_RebootNodeList, nid, "");
    LogText(LL_NOTICE, "The fail reboot node is" .. g_UpdState.g_RebootNodeList);
    SaveState(g_UpdState);
    return 0;
end

--------------------------------------------------------------------------------
-- Func Name       : SetBackupFlowSucc
-- Description     : 重启失败节点流程结束,设置备份的流程状态以及清空重启失败列表
-- Caution         : 由OMM维护, 请勿修改
-- Input           : 无
-- Output          : 无
-- Return          : 无
--------------------------------------------------------------------------------
function SetBackupFlowSucc(state)
    LogText(LL_NOTICE, "The update flow state is" .. state);
    g_UpdState.g_BackupFlowResult = state;
    g_UpdState.g_RebootNodeList = "";
    LogText(LL_NOTICE, "The backup flow state is" .. g_UpdState.g_BackupFlowResult);
    SaveState(g_UpdState);
    return;
end

--------------------------------------------------------------------------------
-- Func Name       : SetBackupFlowInfo
-- Description     : 设置保存的手动执行流程的执行结果
-- Caution         : 由OMM维护, 请勿修改
-- Input           : 无
-- Output          : 无
-- Return          : 无
--------------------------------------------------------------------------------
function SetBackupFlowInfo()
    --手动执行的流程才需要保存结果，自动执行的流程直接跳过
    if (("SyncUpgrade" == g_UpdState.Flow)
    or (nil ~= string.find(g_UpdState.Flow, "Insert"))) then
        return;
    end

    g_UpdState.g_BackupFlowName = g_UpdState.Flow;

    if(0 == #g_UpdState.FailNodeList) then
        g_UpdState.g_BackupFlowResult = BACKUP_FLOW_STATE_SUCC;
    else
        g_UpdState.g_BackupFlowResult = BACKUP_FLOW_STATE_FAIL;
    end

    SaveState(g_UpdState);

    return;
end

--------------------------------------------------------------------------------
-- Func Name       : ClearBackupFlowInfo
-- Description     : 清除保存的手动执行流程的执行结果
-- Caution         : 由OMM维护, 请勿修改
-- Input           : 无
-- Output          : 无
-- Return          : 无
--------------------------------------------------------------------------------
function ClearBackupFlowInfo()
    g_UpdState.g_BackupFlowName = "";
    g_UpdState.g_BackupFlowResult = BACKUP_FLOW_STATE_BUTT;
    g_UpdState.g_RebootNodeList = "";
    SaveState(g_UpdState);

    return;
end

--------------------------------------------------------------------------------
-- Func Name       : Serialize
-- Description     : 将Lua变量序列化为字符串
-- Caution         : 由OMM维护, 请勿修改
-- Input           : o: Lua变量，支持number/string/boolean/tabl/nil这几种类型
-- Output          :
-- Return          : 返回变量序列化之后的字符串
--------------------------------------------------------------------------------
function Serialize(o)
    local s = "";
    local t = type(o);
    if (t == "number") then
        s = s .. o;
    elseif (t == "string") then
        s = s .. string.format("%q", o);
    elseif (t == "boolean") then
        s = s .. tostring(o);
    elseif (t == "table") then
        s = s .. "{\n";
        for k,v in pairs(o) do
            s = s .. " [" .. Serialize(k) .. "] = " .. Serialize(v) .. ",\n";
        end

        s = s .. "}\n";
    elseif t == nil then
        s = nil;
    else
        s = nil;
        LogText(LL_ERROR, "Cannot serialize type: " .. type(o));
    end

    return s;
end

-- TODO: 补全注释
--------------------------------------------------------------------------------
-- Func Name       : HandleAtomReportInfo
-- Description     : HVS不使用这个函数，需要打桩
-- Caution         : 由OMM维护, 请勿修改
-- Input           :
-- Output          :
-- Return          : 此函数不用,直接返回成功
--------------------------------------------------------------------------------
function HandleAtomReportInfo(AtomName, ErrorCode, Progress,
        ExecResult, NodeIp, AtomgGroup, StartTime, EndTime)
    return 0;
end

--------------------------------------------------------------------------------
-- Func Name       : IsRbkAG
-- Description     : 判断是升级的原子组还是回退的
--------------------------------------------------------------------------------
function IsRbkAG(AtomGroup)
    LogText(LL_NOTICE, "The Atomgroup name is " .. AtomGroup .. "and the flow is " .. g_UpdState.Flow)

    for i = 1, #g_UpdateFlowsTbl[g_UpdState.Flow].upd do
        if (g_UpdateFlowsTbl[g_UpdState.Flow].upd[i].AtomGroupName == AtomGroup) then
            LogText(LL_NOTICE, "It is upd atomgroup.")
            return FALSE;
        end
    end
    LogText(LL_NOTICE, "It is rbk atomgroup.")
    return TRUE;
end

-- TODO: 补全注释
--------------------------------------------------------------------------------
-- Func Name       : SaveState
-- Description     : 持久化升级状态到文件中
-- Caution         : 由OMM维护, 请勿修改
-- Input           : 无
-- Output          : 无
-- Return          : 无
--------------------------------------------------------------------------------
function SaveState()
    if (nil == g_UpdState) then
        LogText(LL_ERROR, "The state table is nil.");
        return;
    end

    local s = Serialize(g_UpdState);
    LogText(LL_NOTICE, "Serialized upd state:\n" .. s);
    local fn = "/startup_disk/conf/conf_local/upd_state_flows.lua";

    --以只读打开文件来判断文件的存在性
    local is_exist = false;
    local ff = io.open(fn, "r");
    if ( ff ~= nil ) then
        ff:close();
        is_exist = true;
    end

    --以写+方式打开文件
    local f = io.open(fn, "w+");
    if (nil == f) then
        --解决启动盘短暂修复
        os.execute("sleep " .. 3);
        f = io.open(fn, "w+");
        if (nil == f) then
            LogText(LL_WARN, "Can not open flows state file.");
            return;
        end
    end

    --文件不存在创建新文件时，修改文件权限为644
    if ( is_exist == false ) then
        os.execute("chmod 664 " .. fn);
    end

    f:write(s);
    f:flush();
    f:close();
    os.execute("fsync " .. fn);--2017/9/4
    return SyncState("upd_state_flows.lua");
end

-- TODO: 补全注释
--------------------------------------------------------------------------------
-- Func Name       : LoadState
-- Description     : 从文件中加载升级状态
-- Caution         : 由OMM维护, 请勿修改
-- Input           : 无
-- Output          : 无
-- Return          : 无
--------------------------------------------------------------------------------
function LoadState()
    local fn = "/startup_disk/conf/conf_local/upd_state_flows.lua";
    local f = io.open(fn, "r");
    if (nil == f) then
        --解决启动盘短暂修复
        os.execute("sleep " .. 3);
        f = io.open(fn, "r");
        if (nil == f) then
            LogText(LL_WARN, "Can not read flows state file.");
            return;
        end
    end

    local s = f:read("*all");
    f:close();
    if (s == nil) or (s == "") then
        LogText(LL_WARN, "Flow state file is blank.");
        return;
    end

    s = "return " .. s;
    local func = loadstring(s);
    if (nil == func) then
        LogText(LL_WARN, "No content in flows state file.");
        return;
    end
    g_UpdState = func();
end

-- TODO: 补全注释
--------------------------------------------------------------------------------
-- Func Name       : ReplaceParam
-- Description     : 替换命令行中的参数
-- Caution         : 由OMM维护, 请勿修改
-- Input           : 需要替换的命令
-- Output          : 替换后的命令
-- Return          : 无
--------------------------------------------------------------------------------
function ReplaceParam(Cmd)
    LogText(LL_NOTICE, "Replace param for cmd: " .. Cmd);

    local p = "";
    local W = "";
    for w in string.gmatch(Cmd, "%$%b()") do                --匹配参数: $()
        W = string.gsub(w, "%W", "%%%1");                   --转义参数中的特殊字符
        p = string.sub(w, 3, -2);
        Cmd = string.gsub(Cmd, W, g_UpdState.ParamList[p]); --替换参数
    end

    LogText(LL_NOTICE, "Replaced cmd: " .. Cmd);
end


--================== UPD Framwork API ==========================================
--------------------------------------------------------------------------------
-- Func Name       : SplitStr
-- Description     : 根据分隔符szSeparator，分割字符串szFullString
-- Caution         : 无
-- Input           : szFullString: 被分割的字符串
--                   szSeparator: 分隔符
--
-- Output          : 无
-- Return          : 无
--------------------------------------------------------------------------------
function SplitStr(szFullString, szSeparator)
local nFindStartIndex = 1
local nSplitIndex = 1
local nSplitArray = {}
while true do
   local nFindLastIndex = string.find(szFullString, szSeparator, nFindStartIndex)
   if not nFindLastIndex then
    nSplitArray[nSplitIndex] = string.sub(szFullString, nFindStartIndex, string.len(szFullString))
    break
   end
   nSplitArray[nSplitIndex] = string.sub(szFullString, nFindStartIndex, nFindLastIndex - 1)
   nFindStartIndex = nFindLastIndex + string.len(szSeparator)
   nSplitIndex = nSplitIndex + 1
end
    return nSplitArray
end

--------------------------------------------------------------------------------
-- Func Name       : GetBatchNum
-- Description     : 获取AG暂停批次
-- Caution         : 无
-- Input           : 无
--
-- Output          : 无
-- Return          : 暂停批次
--------------------------------------------------------------------------------
function GetBatchNum(Str, Separator)
    local Num_split=SplitStr(Str, Separator);
    return Num_split[1];
end

--------------------------------------------------------------------------------
-- Func Name       : GetAGIndex
-- Description     : 获取AG暂停Index
-- Caution         : 更新节点属性后, 需要调用SaveState保存;当前节点可能不在NodeList中, 需要遍历MasterList
-- Input           : 无
--
-- Output          : 无
-- Return          : 暂停Index
--------------------------------------------------------------------------------
function GetAGIndex(Str, Separator)
    local Index_split=SplitStr(Str, Separator);
    return Index_split[2];
end

--------------------------------------------------------------------------------
-- Func Name       : IsNeedSetPauseFlag
-- Description     : 根据CurNodeList中节点个数，判断当前是否所有节点都已经上报暂停状态
-- Caution         : 无
-- Input           : 当前CurNodeList
--
-- Output          : 无
-- Return          : true:需要设置 false:不需要设置
--------------------------------------------------------------------------------
function IsNeedSetPauseFlag(NodeList)
    local nodeNum = 0;
    for Index, _ in pairs(NodeList) do
        nodeNum = nodeNum +1;
    end

    if (nodeNum == g_UpdState.g_CurNodeNumForPause + 1 ) then
        return true;
    end
    return false;
end

-- TODO: 补全注释
--------------------------------------------------------------------------------
-- Func Name       : NodeExecAtomGroup
-- Description     : 更新节点的rt,ag属性, 调用升级框架注册给lua的函数ExecAtomGroup通知节点执行指定的AtomGroup
-- Caution         : 更新节点属性后, 需要调用SaveState保存;当前节点可能不在NodeList中, 需要遍历MasterList
-- Input           : Node: table 执行升级的节点属性列表
--                   AtomGroup: table 要执行的原子组列表
--                   ParaTable: table 执行原子组时使用的参数列表
--
-- Output          : 无
-- Return          : 无
--------------------------------------------------------------------------------
function NodeExecAtomGroup(Node, AtomGroup, ParaTable)
    LogText(LL_NOTICE, "NodeExecAtomGroup Node.ip and AtomGroup.AtomGroupName:" .. Node.ip .. AtomGroup.AtomGroupName);
    --优先更新NodeList中节点的状态,包含集群主在NodeList中的场景
    local NodeTab = g_UpdState.NodeList[ip2int(Node.ip)];
    if (NodeTab ~= nil ) then
        NodeTab.rt = ATOM_GROUP_EXEC_RET_INIT
        NodeTab.ag = AtomGroup.AtomGroupName
        SaveState(g_UpdState);
    else
    --此处针对主控不在NodeList中的场景,目前只有恢复上电的场景使用
        NodeTab = g_UpdState.MasterList[ip2int(Node.ip)];
        if (NodeTab ~= nil ) then
            LogText(LL_NOTICE, "Node is in MasterList Node.ip:" .. Node.ip);
            NodeTab.rt = ATOM_GROUP_EXEC_RET_INIT
            NodeTab.ag = AtomGroup.AtomGroupName
            SaveState(g_UpdState);
        end
    end
    LogText(LL_NOTICE, "----NodeTab NULL-----ip:" .. Node.ip);

    if (0 == g_UpdState.FlowUpdState) then --升级状态才去判断是否需要暂停
        if (nil ~= g_UpdState.AutoSuspendAtomGroupInfo) then --解析工具下发字符串合法性
            if (0 ~= string.len(g_UpdState.AutoSuspendAtomGroupInfo)) then
                local PauseAGTable = SplitStr(g_UpdState.AutoSuspendAtomGroupInfo, ",");
                for index=1, #PauseAGTable do
                    --判断是否暂停
                    if (true == g_UpdState.AGPauseInfoArr[PauseAGTable[index]]) then
                        local TmpBatchNum = GetBatchNum(PauseAGTable[index], ":");
                        TmpBatchNum = tonumber(TmpBatchNum);
                        local TmpAGIndex = GetAGIndex(PauseAGTable[index], ":");
                        TmpAGIndex = tonumber(TmpAGIndex);
                        --只支持上传升级包（系统软件）、升级系统、重启系统前暂停，分别对应序号2、5、6。
                        if ((g_UpdState.BatchNo == TmpBatchNum) and ((2 == TmpAGIndex) or (5 == TmpAGIndex) or (6 == TmpAGIndex))) then
                            if (g_UpdateFlowsTbl[g_UpdState.Flow].upd[TmpAGIndex].AtomGroupName == AtomGroup.AtomGroupName) then
                                LogText(LL_NOTICE, "AG is paused and Exec ReportAtomGroup for pausing.");
                                --根据CurNodeList，判断是否需要置暂停标识
                                if (IsNeedSetPauseFlag(g_UpdState.CurNodeList)) then
                                    g_UpdState.AGPauseInfoArr[PauseAGTable[index]] = false;
                                    --置AG暂停标识
                                    g_UpdState.g_AGPauseFalg = 1;
                                end
                                g_UpdState.g_CurNodeNumForPause = g_UpdState.g_CurNodeNumForPause +1;
                                if (1 == g_UpdState.g_AGPauseFalg) then
                                    g_UpdState.g_CurNodeNumForPause = 0;
                                end
                                ReportAtomGroup{flow_id=g_UpdState.FlowId, atom_group=AtomGroup.AtomGroupName, node_ip=Node.ip, result=ATOM_GROUP_EXEC_RET_PAUSE};
                                return;
                            end
                        end
                    end
                end
            end
        end
    end

    local iRet = ExecAtomGroup{flow_id=g_UpdState.FlowId, atom_group=AtomGroup.AtomGroupName, node_ip=Node.ip, over_time=AtomGroup.timeout, param=ParaTable};
    if (RET_ERR == iRet) then
        LogText(LL_ERROR, "----ExecAtomGroup Fail and Server ReportAtomGroup.");
        ReportAtomGroup{flow_id=g_UpdState.FlowId, atom_group=AtomGroup.AtomGroupName, node_ip=Node.ip, result=ATOM_GROUP_EXEC_RET_TIMEOUT};
    end
end

--------------------------------------------------------------------------------
-- Func Name       : SplitIPList
-- Description     : 执行版本同步之前分割由某个分隔符连接的多个ip字符串，分成单个ip后存放在一个table中
-- Caution         : ip的格式为 255.255.255.255
-- Input           : Iplist: string 多个ip组成的字符串
--                   Separator: string 分割ip的分隔符
--
-- Output          : 无
-- Return          : 无
--------------------------------------------------------------------------------
function SplitIPList(Iplist, Separator)
    local SplitIpArray = {};
    local s = "";
    --HVS构造ip列表格式为：126.126.126.0,126.126.126.1最后不含有分割号所以此处添加一个分割号
    Iplist = Iplist .. Separator
    for w in string.gmatch(Iplist, "%d+.-,") do
        s = string.gsub(w, "," , "");
        table.insert(SplitIpArray, s);
    end
    return SplitIpArray
end

--------------------------------------------------------------------------------
-- Func Name       : GetMsgRetryInfo
-- Description     : 获取发送消息的重试时间和重试间隔
-- Caution         : 依次返回重试的间隔 重试的次数
-- Input           : 无
-- Output          : 15:重试的次数 10:重试的间隔
-- Return          : 无
--------------------------------------------------------------------------------
function GetMsgRetryInfo()
    return 15, 10;
end

--------------------------------------------------------------------------------
-- Func Name       : GetNidByIp
-- Description     : 根据ip获取nid
-- Caution         : 传入节点的ip得出节点的nid
--                   g_UpdState.g_ClusterInfo中目前只是保存了节点的ip1
-- Input           : ip string 节点的ip
-- Output          : nid string 节点的nid
-- Return          : nid string 节点的nid
--------------------------------------------------------------------------------
function GetNidByIp(Ip)
    LogText(LL_NOTICE, "GetNidByIp" .. Ip );
    for Index, Node in pairs(g_UpdState.g_ClusterInfo) do
        if (Node.ip == Ip) then
            LogText(LL_NOTICE, "GetNidByIp" .. Node.Nid);
            return  Node.Nid;
        end
    end
    LogText(LL_NOTICE, "GetNidByIp has a wrong Ip" .. Ip );
    return nil;
end

--------------------------------------------------------------------------------
-- Func Name       : IsMasterNode
-- Description     : 根据ip获取判断节点是否为主控
-- Caution         : 当发生主备倒换后会实时更新g_MasterSlave,所以要获取主控相关的信息都要以g_MasterSlave为主
-- Input           : ip string 节点的ip
-- Output          : true  bool 节点是主控
--                   false bool 节点不是主控
-- Return          : true  bool 节点是主控
--                   false bool 节点不是主控
--------------------------------------------------------------------------------
function IsMasterNode(Ip)
    LogText(LL_NOTICE, "IsMaster" .. Ip );
    for Index, Node in pairs(g_UpdState.g_MasterSlave) do
        if ("1" == Node.IsMaster and Ip == Node.ip) then
            LogText(LL_NOTICE, "Master ip is" .. Node.ip);
            return  true;
        end
    end
    LogText(LL_NOTICE, "The node is not master.");
    return false;
end

--------------------------------------------------------------------------------
-- Func Name       : GetSuccNodeNid
-- Description     : 从指定的列表中获取升级成功或回退成功的节点列表
-- Caution         : 在线升级过程中，获取本批次需要重启的节点;
--                   离线升级失败后回退及在线升级失败后回退重启时，直接重启本批次
-- Input           : 无
-- Output          : 12:重试的间隔 10:重试的次数
-- Return          : 无
--------------------------------------------------------------------------------
function GetSuccNodeNid(NodeList)
    LogText(LL_NOTICE, "Exec GetSuccNodeNid. " );
    local SuccNodeL = "";
    local TmpNid;
    if(0 == g_UpdState.FlowUpdState)then
        for SuccIndex, SuccNode in pairs(g_UpdState.SuccNodeList) do
            for Index, Node in pairs(NodeList) do
                if (SuccNode.ip == Node.ip) then
                    TmpNid = GetNidByIp(SuccNode.ip);
                    if(nil ~= TmpNid)then
                        SuccNodeL = SuccNodeL .. TmpNid .. "," ;
                    end
                    break;
                end
            end
        end
    else
    --离线回退时重启所有节点传入的NodeList为当前回退的节点列表g_UpdState.CurNodeList
    --在线升级时也是回退当前批次的所有节点重启当前批次所有节点传入的NodeList为当前回退的节点列表g_UpdState.CurNodeList
        for Index, Node in pairs(NodeList) do
            TmpNid = GetNidByIp(Node.ip);
            if(nil ~= TmpNid)then
                SuccNodeL = SuccNodeL .. TmpNid .. "," ;
            end
        end
    end
    LogText(LL_NOTICE, "Exec GetCurBatchSuccNid return SuccNodeList: " .. SuccNodeL);
    return SuccNodeL;
end

--------------------------------------------------------------------------------
-- Func Name       : ManagerRunAtomGroup
-- Description     : 只在集群主执行的原子组实现
-- Caution         : 当前的流程在主控执行此原子组且原子组执行失败后结束flow
--                 : 回调系统控制及离线准备获取成功节点列表时传入当前节点列表CurNodeList
-- Input           : AtomGroup table 执行的原子组信息表
-- Output          : 无
-- Return          : 无
--------------------------------------------------------------------------------
function ManagerRunAtomGroup(AtomGroup)
    LogText(LL_NOTICE, "ManagerRunAtomGroup: " .. AtomGroup.AtomGroupName);

    local nidlist = GetSuccNodeNid(g_UpdState.CurNodeList);
    g_UpdState.ParamList["nidlist"] = nidlist;

    if (g_UpdState.Flow == "OnlineUpgrade" or g_UpdState.Flow == "OfflineUpgrade") then
        g_UpdState.ParamList["version"] = g_UpdState.ParamList.pkgversion;
    end

    for _, Node in pairs(g_UpdState.MasterList) do
        Node.state = ATOM_GROUP_EXEC_RET_INIT; --设置MasterList节点中的节点状态为未执行
        NodeExecAtomGroup(Node, AtomGroup, Table2String(g_UpdState.ParamList));
    end
end

--------------------------------------------------------------------------------
-- Func Name       : SaveNodeIP
-- Description     : 保存集群中所有节点的ip及nid信息
-- Caution         : 传入集群节点信息构成的字符串
-- Input           : AtomGroup table 执行的原子组信息表
-- Output          : 无
-- Return          : 无
--------------------------------------------------------------------------------
function SaveNodeIP(NidIpInfo)
    LogText(LL_NOTICE, "Save Node IP " );
    local func = loadstring("return " .. NidIpInfo); --将字符串转换成table
    g_UpdState.g_ClusterInfo = func();
    SaveState(g_UpdState);
    return 0;
end

--------------------------------------------------------------------------------
-- Func Name       : IsAbnormalSwapMaster
-- Description     : 判断是否发生异常主备倒换
-- Caution         : 当前只处理在线升级第一批次和第二批次升级时的主备倒换，主备倒换后走回退流程
-- Input           : 无
-- Return          : false 未发生主备倒换 true 发生了主备倒换
--------------------------------------------------------------------------------
function IsAbnormalSwapMaster()
    local AtomGroup;
    LogText(LL_NOTICE, "Enter IsAbnormalSwapMaster." );
    if(0 == g_UpdState.g_SwapMasterFalg) then
        LogText(LL_NOTICE, "Master not swap.");
        g_UpdState.ParamList.continue = CON_TYPE_START;
        return false;
    end
    AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.AtomGroupIndex];
    if("OnlineUpgrade" == g_UpdState.Flow and 0 == g_UpdState.FlowUpdState)then
        ---非最后一批发生主备倒换后，需要回退，其他情况不回退
        if((1 == g_UpdState.BatchNo) or (0 == g_UpdState.g_FinishAllBatch))then
            if ((g_UpdState.ParamList.continue == CON_TYPE_RETRY) and (AtomGroup.AtomGroupName == "AgentPrepareForOnlineUpgrade")) then
                LogText(LL_NOTICE, "No exec abnormal master swap.");
                g_UpdState.g_SwapMasterFalg = 0;
                g_UpdState.ParamList.continue = CON_TYPE_START;
                return false;
            end
            LogText(LL_NOTICE, "Exec abnormal master swap.");
            return true;
        end
    end
    LogText(LL_NOTICE, "Master swap, but not on online first two batch.");
    return false;
end

--------------------------------------------------------------------------------
-- Func Name       :SetMasterSlaveNode
-- Description     :设置主备节点信息,发生主备倒换后更新MasterList
--------------------------------------------------------------------------------
function SetMasterSlaveNode(MasterSlave)
    LogText(LL_NOTICE, "Set Master Node " );
    local oldMasterNid=GetMasterNid();
    local func = loadstring("return " .. MasterSlave); --将字符串转换成table
    g_UpdState.g_MasterSlave = func();

    local NewMaster = GetMasterNid();
    if((FLOW_STATE_FINISH ~= g_UpdState.FlowState) and (NewMaster ~= oldMasterNid))then
        --升级执行的过程中需要设置主备倒换标记，升级暂停时不需要设置
        if(FLOW_STATE_RUNNING == g_UpdState.FlowState)then
            g_UpdState.g_SwapMasterFalg = 1;
            LogText(LL_NOTICE, "Set SwapMaster Falg 1");
        end
        --升级流程未结束时就需要更新MasterList
        g_UpdState.MasterList = {};
        for _, Node in pairs(g_UpdState.g_MasterSlave) do
            if(Node.IsMaster == "1") then
                ---避免发生多次主备倒换后需要执行只能在主控执行的AG
                g_UpdState.g_MasterIp = Node.ip;
                LogText(LL_NOTICE, "Set new master list and the g_UpdState.FlowState is:" .. g_UpdState.g_MasterIp ..g_UpdState.FlowState);
                g_UpdState.MasterList[ip2int(Node.ip)] = {ip=Node.ip,type="Manager",
                state=ATOM_GROUP_EXEC_RET_INIT,rt=ATOM_GROUP_EXEC_RET_INIT,bt=-1,et=-1,pr=0,batchno=0,ag=""};
            end
        end
    end
    --进程初始化是在所有的进程都初始化完成后才会处理相关的任务，所以初始化时不能调用SaveState
    --SaveState(g_UpdState);
    return 0;
end

--------------------------------------------------------------------------------
-- Func Name       :ReportFault
-- Description     :模拟故障节点给自己发原子组执行失败的消息
--------------------------------------------------------------------------------
function ReportFault(NodeIp)
    LogText(LL_NOTICE, "Report Fault, ip is " .. NodeIp);
    ReportAtomGroup{flow_id=g_UpdState.FlowId, atom_group="SyncSysAndFirmware", node_ip=NodeIp, result=ATOM_GROUP_EXEC_RET_FAIL};
    return 0;
end

--------------------------------------------------------------------------------
-- Func Name       : TransFlowName
-- Description     : 转换流程名称, 用于将工具界面传递过来的流程名称转换为
--                   流程文件中流程名称, 产品根据实际情况实现
-- Caution         : 由升级框架调用，函数名称、参数、返回值必须与模板一致
-- Input           : FlowName: string  流程名称
-- Output          : string 转换后的流程名称
--------------------------------------------------------------------------------
function TransFlowName(FlowName)
    return FlowName;
end

--导包类型，与C代码UPGRADE_PKGTYPE_E保持一致
function GetDownloadPkgType()
    LogText(LL_NOTICE, "--Exec GetDownloadPkgType---");
    if(nil == g_UpdState.ParamList.pkgpath)then
        return 9;
    end
    
    local filepath = g_UpdState.ParamList.pkgpath;

    local index = string.find(filepath, "SYS_software")
    if(nil ~= index)then
        return 1;
    end
    
    index = string.find(filepath, "hotpatch")
    if(nil ~= index)then
        return 3;
    end
    
    index = string.find(filepath, "diskfw")
    if(nil ~= index)then
        return 4;
    end
    
    index = string.find(filepath, "aslpatch")
    if(nil ~= index)then
        return 5;
    end
    
    index = string.find(filepath, "oem")
    if(nil ~= index)then
        return 6;
    end
    
    return 9;
end
--------------------------------------------------------------------------------
-- Func Name       :NotifyUpdStateChangeEvent
-- Description     :通知EMP升级事件，同步及升级过程中都要通知
--------------------------------------------------------------------------------
function NotifyUpdStateChangeEvent(FlowName, UpdState, UpdType)
    local cmd_NotifyEvent;
    
    if(FlowName == "DownloadPkg")then
        local pkgtype = GetDownloadPkgType();
        cmd_NotifyEvent = "/OSM/bin/upgrade_atom -f NOTIFY_UPGRADE_CHANGE_EVENT" .. " -h " .. FlowName .. " -u " .. UpdState .. " -t " .. UpdType .. " -a " .. pkgtype;
        LogText(LL_NOTICE, "Notify upgrade state change event: " .. cmd_NotifyEvent);
        System(cmd_NotifyEvent, 5);
        return 0;
    end
    if ((nil ~= g_UpdState.ParamList.pkgspcversion) and (nil ~= g_UpdState.ParamList.rollspcversion)) then
        cmd_NotifyEvent = "/OSM/bin/upgrade_atom -f NOTIFY_UPGRADE_CHANGE_EVENT" .. " -h " .. FlowName .. " -u " .. UpdState .. " -t " .. UpdType .. " -v " .. g_UpdState.ParamList.pkgspcversion .. " -e " .. g_UpdState.ParamList.rollspcversion;
    else
        cmd_NotifyEvent = "/OSM/bin/upgrade_atom -f NOTIFY_UPGRADE_CHANGE_EVENT" .. " -h " .. FlowName .. " -u " .. UpdState .. " -t " .. UpdType;
    end
    LogText(LL_NOTICE, "Notify upgrade state change event: " .. cmd_NotifyEvent);
    System(cmd_NotifyEvent, 5);
    return 0;
end

function SetNotifySysState()

    local notifySysUpgradeFinish = 2; local notifySysRollBackFinish = 3;

    if (g_UpdState.g_bHandleFault == 1) then
        LogText(LL_NOTICE, "The version is the old version.");
        g_UpdState.g_bHandleFault = nil;
        return notifySysRollBackFinish,0;
    end

    --还没正式升级，返回回退成功
    if (g_isPreStartFailed == 1 ) then
        LogText(LL_NOTICE, "Notify sys fail in pre start failed.");
        return notifySysRollBackFinish,0;
    end

    if ((g_UpdState.g_FlowStage ~= nil) and (g_UpdState.g_FlowStage.stage == STAGE_FIRST_FLOW) and 0 ~= #g_UpdState.FailNodeList) then
        LogText(LL_NOTICE, "The first flow upgrade failed.");
        return notifySysRollBackFinish,0;
    end

    --升级完成，失败由位图做区分
    if (0 == g_UpdState.FlowUpdState) then
        if (0 == #g_UpdState.FailNodeList) then
            return notifySysUpgradeFinish,0;
        else
            return notifySysUpgradeFinish,GetFailNodeBitMap();
        end
    end

    --回退完成，失败由位图做区分
    if (MANAGER_FLOW_STATE_UPG_RBK == g_UpdState.FlowUpdState) then
        --离线升级交换机失败是重启集群，不会进入fault状态，需要返系统控制回退完成-成功
        local AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.FailRbkAGIndex];
        if(("OfflineUpgrade" == g_UpdState.Flow) and ("UpgradePcieSw" == AtomGroup.AtomGroupName)) then
            LogText(LL_NOTICE, "The offline upgrade fail when UpgradePcieSw fail and notify sys 3 0.");
            return notifySysRollBackFinish,0;
        end
        for _, Node in pairs(g_UpdState.NodeList) do
            if (NODE_STATE_FAIL == Node.st) or (NODE_STATE_TIMEOUT == Node.st) then
                --跨R版本多批次升级失败，回退又失败，主异常切到新版本，需要通知系统控制解禁配
                if (g_isCrossUpgrade == 1) then
                    return notifySysRollBackFinish,0;
                else
                    return notifySysRollBackFinish,GetFailNodeBitMap();
                end
            end
        end
        return notifySysRollBackFinish,0;
    end


    return notifySysUpgradeFinish,0;
end

function GetFailNodeBitMap()

    local bitmap = 0;

    for _, Node in pairs(g_UpdState.FailNodeList) do
        nid = GetNidByIp(Node.ip);
        if ((nid + 0) >= 0 and (nid + 0) <= 16) then
            bitmap = bitmap + 2 ^ nid;
        end
    end

    LogText(LL_NOTICE, "Get fail node bitmap is: " .. bitmap);
    return bitmap;
end

--------------------------------------------------------------------------------
-- Func Name       :NotifySysStartUpgrade
-- Description     :执行升级时，通知系统控制升级开始禁配置或者升级完成取消禁配置
--------------------------------------------------------------------------------
function NotifySysStartUpgrade(upgstate)
    if (upgstate == FLOW_STATE_RUNNING) then
        LogText(LL_NOTICE, "---Exec NotifySysStartUpgrade 0---");
        if ((nil ~= g_UpdState.ParamList.pkgspcversion) and (nil ~= g_UpdState.ParamList.rollspcversion)) then
            cmd = "/OSM/bin/upgrade_atom -f NOTIFY_SYS_UPDATE_STATE -u 0 -a 0 -h " .. g_UpdState.Flow .. " -v " .. g_UpdState.ParamList.pkgspcversion .. " -e " .. g_UpdState.ParamList.rollspcversion;
        else
            cmd = "/OSM/bin/upgrade_atom -f NOTIFY_SYS_UPDATE_STATE -u 0 -a 0 -h " .. g_UpdState.Flow;
        end
        if( nil == string.find(g_UpdState.Flow, "Insert") )then
            NotifyDrvUpdState{upd_state = 1, upd_flowname = g_UpdState.Flow, upd_locationid = "all"};
        end
    else
        local state,bitmap = SetNotifySysState();
        LogText(LL_NOTICE, "---Exec NotifySysStartUpgrade " .. state .. bitmap );
        cmd = "/OSM/bin/upgrade_atom -f NOTIFY_SYS_UPDATE_STATE -u " .. state .. " -h " .. g_UpdState.Flow .. " -a " .. bitmap;
        if( nil == string.find(g_UpdState.Flow, "Insert") )then
            NotifyDrvUpdState{upd_state = 2, upd_flowname = g_UpdState.Flow, upd_locationid = "all"};
        end
    end

    if( ("SyncUpgrade" == g_UpdState.Flow) or (nil ~= string.find(g_UpdState.Flow, "Insert")) )then
        LogText(LL_NOTICE, "The sync upgrade no need notify sys.");
        return 0;
    end
    return System(cmd, 16);
end

--------------------------------------------------------------------------------
-- Func Name       :IsFlowNeedNotify
-- Description     :判断是否需要通知系统控制升级事件及通知EMP事件
--------------------------------------------------------------------------------
function IsFlowNeedNotifySys(Flow)
    local flowsNeedNotifySys = {["OfflineUpgrade"]=true, ["OnlineUpgrade"]=true,
                             ["HotPatchUpgrade"]=false, ["HotPatchRollback"]=false, ["HotPatchDelete"]=false,
                             ["SyncUpgrade"]=true, ["InsertFRM"]=true, ["InsertBBU"]=true,
                             ["InsertSAS"]=true, ["InsertFCoE"]=true, ["InsertTOE"]=true,
                             ["InsertPCIE"]=true, ["InsertPCIESW"]=true, ["AslPatchUpgrade"]=false,
                             ["InsertMGMT"]=true, ["InsertIXGBE"]=true, ["Insert10GBaseT"]=true,
                             ["InsertSmartIO"]=true, ["InsertIOCHD"]=true, ["InsertIOC10GE"]=true,["InsertSmartIOInOfflineStatus"]=true,
                             ["Insert12GSAS"]=true, ["Insert16GFC"]=true, ["Insert56GIB"]=true, ["InsertACCFPGA"]=true,
                             ["InsertIOC8G"]=true, ["InsertIOC16G"]=true, ["InsertSHARECARD"]=true,
                             ["InsertBBUBackGround"]=true, ["InsertSHARECARD12GSAS"]=true, ["InsertMGMTBackGround"]=true};

    if(flowsNeedNotifySys[Flow])then
        LogText(LL_NOTICE, "The flow need notify sysctrl:" .. Flow);
        return true;
    end
    LogText(LL_NOTICE, "The flow no need notify sysctrl:" .. Flow);
    return false;
end


--------------------------------------------------------------------------------
-- Func Name       :PublishUpdStateChgEvent
-- Description     :执行升级时，通知事件中心状态变化
--------------------------------------------------------------------------------
function PublishUpdStateChgEvent(upgstate)
    if (upgstate == FLOW_STATE_RUNNING) then
        LogText(LL_NOTICE, "---Exec PublishUpdStateChgEvent 0---");
        NotifyUpdStateChangeEvent(g_UpdState.Flow, 0, "upgrade");
    else
        LogText(LL_NOTICE, "---Exec PublishUpdStateChgEvent " .. upgstate );
        NotifyUpdStateChangeEvent(g_UpdState.Flow, 1, "upgrade");
    end

    return;
end
--------------------------------------------------------------------------------
-- Func Name       :IsFlowNeedPublishEvent
-- Description     :判断当前流程是否需要通过OM事件中心进行发布
--------------------------------------------------------------------------------
function IsFlowNeedPublishEvent(Flow)
    local flowNeedPublishEvent = {["OfflineUpgrade"]=true, ["OnlineUpgrade"]=true,
                             ["HotPatchUpgrade"]=true, ["HotPatchRollback"]=true, ["HotPatchDelete"]=true,
                             ["SyncUpgrade"]=true, ["InsertFRM"]=true, ["InsertBBU"]=true,
                             ["InsertSAS"]=true, ["InsertFCoE"]=true, ["InsertTOE"]=true,
                             ["InsertPCIE"]=true, ["InsertPCIESW"]=true, ["AslPatchUpgrade"]=false,
                             ["InsertMGMT"]=true, ["InsertIXGBE"]=true, ["Insert10GBaseT"]=true,
                             ["InsertSmartIO"]=true, ["InsertIOCHD"]=true, ["InsertIOC10GE"]=true,["InsertSmartIOInOfflineStatus"]=true,
                             ["Insert12GSAS"]=true, ["Insert16GFC"]=true, ["Insert56GIB"]=true, ["InsertACCFPGA"]=true,
                             ["InsertIOC8G"]=true, ["InsertIOC16G"]=true, ["InsertSHARECARD"]=true,
                             ["InsertBBUBackGround"]=true, ["InsertSHARECARD12GSAS"]=true, ["InsertMGMTBackGround"]=true, ["DownloadPkg"]=true};

    if(flowNeedPublishEvent[Flow])then
        LogText(LL_NOTICE, "The flow need publish event:" .. Flow);
        return true;
    end
    LogText(LL_NOTICE, "The flow no need publish event:" .. Flow);
    return false;
end

--------------------------------------------------------------------------------
-- Func Name       :IsFlowNeedForbidSync
-- Description     :判断当前的升级流程是否禁止同步，升级中禁止执行同步
--------------------------------------------------------------------------------
function IsFlowNeedForbidSync(Flow)
    local flowsNeedForbidSync = {["OfflineUpgrade"]=true, ["OnlineUpgrade"]=true,
                                 ["HotPatchUpgrade"]=true, ["HotPatchRollback"]=true, ["HotPatchDelete"]=true,
                                 ["DownloadPkg"]=true, ["AslPatchUpgrade"]=true};

    if(flowsNeedForbidSync[Flow])then
        LogText(LL_NOTICE, "The flow need forbid sync upgrade:" .. Flow);
        return true;
    end
    LogText(LL_NOTICE, "The flow no need forbid sync upgrade:" .. Flow);
    return false;
end

--------------------------------------------------------------------------------
-- Func Name       :CreateUpgradeStateFile
-- Description     :创建升级状态标记文件，升级中禁止执行同步
--------------------------------------------------------------------------------
function CreateUpgradeStateFile()
    local fn = "/startup_disk/conf/conf_local/state_upd.lua";
    local f = io.open(fn, "w+");
    if (nil == f) then
        os.execute("sleep " .. 3);
        f = io.open(fn, "w+");
        if (nil == f) then
            return 1;
        end
    end
    os.execute("chmod 664 " .. fn);
    f:write(g_UpdState.Flow);
    f:flush();
    f:close();
    os.execute("fsync " .. fn);
    return 0;
end

--------------------------------------------------------------------------------
-- Func Name       :CreateUpdStateForbidSync
-- Description     :在所有的节点产生升级状态标记文件，当此文件存在时禁止执行同步
--------------------------------------------------------------------------------
function CreateUpdStateForbidSync()
    LogText(LL_NOTICE, "Create upgrading state file." );
    if(1 == CreateUpgradeStateFile()) then
        LogText(LL_WARN, "Fail to create the upgrade state file.");
        return 1;
    end

    --向升级的所有节点同步升级状态标记文件
    for Index, Node in pairs(g_UpdState.NodeList) do
        if (not IsMasterNode(Node.ip)) then
            LogText(LL_NOTICE, "Exec SyncUpgradingStateFile.");
            local file = "/startup_disk/conf/conf_local/state_upd.lua";
            local filepath = "/startup_disk/conf/conf_local";
            local iRet=TransSomethingByRealIP{virtual_ip = Node.ip, trans_type = "PUT_FILE", src = file, dst = filepath, timeout=5};
            if(0 ~= iRet) then
                LogText(LL_NOTICE, "Create upgrade state file fail the ip: " .. Node.ip .. "the result: " .. iRet);
                ServerSendMsgToAgentExecCmd{node_ip = Node.ip, exec_cmd = "/OSM/script/upgrade_preck.sh -q"};
                iRet = TransSomethingByRealIP{virtual_ip = Node.ip, trans_type = "PUT_FILE", src = file, dst = filepath, timeout=5};
                if(0 ~= iRet) then
                    LogText(LL_NOTICE, "Create upgrade state file fail the ip: " .. Node.ip .. "the result: " .. iRet);
                    ServerSendMsgToAgentExecCmd{node_ip = Node.ip, exec_cmd = "/OSM/script/upgrade_preck.sh -q"};
                    return 1;
                end
            end
        end
    end
    return 0;
end

--------------------------------------------------------------------------------
-- Func Name       :DelRescuePkgForSync
-- Description     :解决备件因盘空间不够同步失败
--------------------------------------------------------------------------------
function DelRescuePkgForSync()
    LogText(LL_NOTICE, "Del rescue pkg ForSync." );
    if ("1" == g_UpdState.ParamList.syncflag)then
        for Index, Node in pairs(g_UpdState.CurNodeList) do
            --设置可写后删除
            ServerSendMsgToAgentExecCmd{node_ip = Node.ip, exec_cmd = "/OSM/script/os_system_update.sh -w"};
            ServerSendMsgToAgentExecCmd{node_ip = Node.ip, exec_cmd = "rm -rf /startup_disk/image/boot_rescue/update_pkg"};
            ServerSendMsgToAgentExecCmd{node_ip = Node.ip, exec_cmd = "rm -f /startup_disk/image/boot_rescue/initrd"}; 
            ServerSendMsgToAgentExecCmd{node_ip = Node.ip, exec_cmd = "ln -s ../boot_osp/initrd /startup_disk/image/boot_rescue/initrd"}; 
            ServerSendMsgToAgentExecCmd{node_ip = Node.ip, exec_cmd = "rm -f /startup_disk/image/boot_rescue/bzImage"}; 
            ServerSendMsgToAgentExecCmd{node_ip = Node.ip, exec_cmd = "ln -s ../boot_osp/bzImage /startup_disk/image/boot_rescue/bzImage"}; 
        end
    end

    return 0;
end
--------------------------------------------------------------------------------
-- Func Name       :DeleteUpdStateAllowSync
-- Description     :升级流程结束后删除升级状态标记文件，当此文件不存在时允许同步
--------------------------------------------------------------------------------
function DeleteUpdStateAllowSync(Flow)
    LogText(LL_NOTICE, "Delete upgrading state file." );
    local fn = "/startup_disk/conf/conf_local/state_upd.lua";
    local rm = os.remove(fn);
    if (nil == rm) then
        LogText(LL_WARN, "Delete upgrade state file fail.");
        --为了避免主控删除失败后不会继续删除其他控制器的文件，这里删除失败不返回
    --return;
    end
    --删除所有节点的升级状态文件及清空/OSM/update,下载包流程结束后不可以清空
    if(Flow == "DownloadPkg")then
        for Index, Node in pairs(g_UpdState.NodeList) do
            if (not IsMasterNode(Node.ip)) then
                LogText(LL_NOTICE, "--DownloadPkg Exec DeleteStateFile---");
                ServerSendMsgToAgentExecCmd{node_ip = Node.ip, exec_cmd = "/OSM/script/upgrade_preck.sh -q"};
            end
        end
        return;
    end
    local cmd = "/OSM/script/os_system_update.sh -k /OSM/update";
    LogText(LL_NOTICE, "Flow finish and delete /OSM/update and upgrade state file." .. Flow);
    --清除集群主上内存文件
    System(cmd, 3);
    for Index, Node in pairs(g_UpdState.NodeList) do
        if (not IsMasterNode(Node.ip)) then
            LogText(LL_NOTICE, "--Exec DeleteStateFile---");
            ServerSendMsgToAgentExecCmd{node_ip = Node.ip, exec_cmd = "/OSM/script/os_system_update.sh -k /OSM/update"};
        end
    end
    return;
end

--------------------------------------------------------------------------------
-- Func Name       :PreHandleBeforeUpdStart
-- Description     :开始升级流程前的预处理，主要包括通知系统控制及产生升级状态文件
--------------------------------------------------------------------------------
function PreHandleBeforeUpdStart(Flow)
    LogText(LL_NOTICE, "Exec PreHandleBeforeUpdStart Flow:" .. Flow);

    local needForbidSync = IsFlowNeedForbidSync(Flow);
    local needNotifySys = IsFlowNeedNotifySys(Flow);
    local needPublishEvent = IsFlowNeedPublishEvent(Flow);
    local iRet;

    if(needForbidSync)then
        iRet = CreateUpdStateForbidSync();
        --创建升级状态文件失败，直接终止流程，以免影响后续同步
        if(0 ~= iRet) then
            LogText(LL_ERROR, "Create upgrade state file failed.");
            return EBADF;
        end
    end
    --广播升级时间和通知系统控制顺序不能调整(见DTS2017022802762)
    if(needPublishEvent)then
        PublishUpdStateChgEvent(FLOW_STATE_RUNNING);
    end
    
    if(needNotifySys)then
        local ret = NotifySysStartUpgrade(FLOW_STATE_RUNNING);
        if(0 ~= ret)then
            LogText(LL_ERROR, "Exec NotifySysStartUpgrade fail.");
            g_isPreStartFailed = 1;
            return EPERM;
        end
    end
    
    if (MODE_SHIELD_ALARM == g_UpdState.ParamList.shieldalarmtype) then
        NotifyAlarmShield(true);              --升级开始通知alarm模块屏蔽告警
    end
    return RET_OK;
end

--------------------------------------------------------------------------------
-- Func Name       :PostHandleAfterUpdFinish
-- Description     :结束升级流程前的预处理，主要包括通知系统控制及删除升级状态文件
--------------------------------------------------------------------------------
function PostHandleAfterUpdFinish(Flow)
    LogText(LL_NOTICE, "Exec PostHandleAfterUpdFinish Flow:" .. Flow);

    local needNotifySys = IsFlowNeedNotifySys(Flow);
    local needPublishEvent = IsFlowNeedPublishEvent(Flow);
    local needForbidSync = IsFlowNeedForbidSync(Flow);
    --广播升级事件和通知系统控制顺序不能调整(见DTS2017022802762)
    if(needPublishEvent)then
        PublishUpdStateChgEvent(FLOW_STATE_FINISH);
    end   
	
    if(needNotifySys)then
        local ret = NotifySysStartUpgrade(FLOW_STATE_FINISH);
        if(0 ~= ret)then
            LogText(LL_ERROR, "Exec NotifySysFinishUpgrade fail.");
            ReportAbnormalAGInfo{atom_group="NotifySysUpgradeFinish", node_ip=GetMasterIp(), error_code=3, result=1};
            -- 升级成功的情况下通知SYS失败 升级流程结束 直接进fault状态
            if ((g_UpdState.Flow == "OnlineUpgrade" or g_UpdState.Flow == "OfflineUpgrade") and IsAllNodeSuccInNodeList()) then
                LogText(LL_NOTICE, "Upgrade success but notify sys failed, enter the fault state.");
                CreateFileForFault();
                return;
            end
        end
    end
    
    if (MODE_SHIELD_ALARM == g_UpdState.ParamList.shieldalarmtype) then
        NotifyAlarmShield(false);              --升级结束通知alarm解除告警屏蔽
    end
    
    if(needForbidSync)then
        DeleteUpdStateAllowSync(Flow);
    end
    --2800单控升级完成后备份boot_osp目录到/OSM/ccdb/soft_backup
    local ret = BackupBootOSP();
    if(0 ~= ret)then
        LogText(LL_WARN, "Backup boot_osp failed.");
    end

    --记录升级历史路径到/OSM/ccdb/store/upgrade_path.log
    local ret = RecordUpgradePath();
    if(0 ~= ret)then
        LogText(LL_WARN, "Record upgrade path failed.");
    end
end

function ReusmeVersionInfo()
    local fn = "/startup_disk/conf/conf_local/state_version.lua";
    local f = io.open(fn, "r");

    if (nil == f) then
        LogText(LL_WARN, "Can not read state version file.");
        return;
    end

    if (g_UpdState.Flow == "DownloadPkg") then
        LogText(LL_NOTICE, "Delete state_version.lua when start a flow.");
        DeleteFileForFault();
        return;
    end

    local s = f:read("*all");
    f:close();
    if (s == nil) or (s == "") then
        LogText(LL_WARN, "Version state file is blank.");
        return;
    end

    s = "return " .. s;
    local func = loadstring(s);
    if (nil == func) then
        LogText(LL_WARN, "No content in flows state file.");
        return;
    end
    local verinfo = func();

    --fault状态下，系统是升级前的版本，可能前面回退失败，处理后已经是老版本,+0规避格式问题
    if ((nil ~= g_UpdState.ParamList.pkgversion) and ((g_UpdState.ParamList.pkgversion+0) == (verinfo.rollversion+0))) then
        g_UpdState.g_bHandleFault = 1;
        DeleteFileForFault();
    end
    g_UpdState.ParamList.pkgversion = verinfo.pkgversion;
    g_UpdState.ParamList.rollversion = verinfo.rollversion;
    LogText(LL_WARN, "Resume pkgversion and rollversion is: " .. g_UpdState.ParamList.pkgversion .. g_UpdState.ParamList.rollversion);
end
--------------------------------------------------------------------------------
-- Func Name       :BackupSpcVesionWhenRepair
-- Description     :Repair时备份SPC版本号
--------------------------------------------------------------------------------
function BackupSpcVesionWhenRepair()
    local fn = "/startup_disk/conf/conf_local/state_version.lua";
    local f = io.open(fn, "r");
    if (nil == f) then
        LogText(LL_WARN, "no need backup spcversion.");
        return;
    end
    LogText(LL_NOTICE, "backup spcversion when repair.");
    REPAIR_PKG_SPCVERSION = g_UpdState.ParamList.pkgspcversion;
    REPAIR_ROLL_SPCVERSION = g_UpdState.ParamList.rollspcversion;
    return;
end
--------------------------------------------------------------------------------
-- Func Name       :RecoverySpcVersion
-- Description     :恢复SCP版本号
--------------------------------------------------------------------------------
function RecoverySpcVersionWhenRepair()
    local fn = "/startup_disk/conf/conf_local/state_version.lua";
    local f = io.open(fn, "r");
    if (nil == f) then
        LogText(LL_WARN, "no need recovery spcversion.");
        return;
    end
    LogText(LL_NOTICE, "recovery spcversion when repair.");
    g_UpdState.ParamList.pkgspcversion = REPAIR_PKG_SPCVERSION;
    g_UpdState.ParamList.rollspcversion = REPAIR_ROLL_SPCVERSION;
    return;
end
--------------------------------------------------------------------------------
-- Func Name       :InitExecFlowInfo
-- Description     :初始化流程执行信息
--------------------------------------------------------------------------------
function InitExecFlowInfo(FlowId, Flow, NodeList, ParamList)
    LogText(LL_NOTICE, "Exec InitUpdTaskInfo.");
    g_UpdState.Flow = Flow;
    g_UpdState.FlowId = FlowId;
    g_UpdState.FlowState = FLOW_STATE_FINISH;
    g_UpdState.FlowUpdState = MANAGER_FLOW_STATE_UPG;
    g_UpdState.BatchNo = 0;
    g_UpdState.RbkBatchNo = 0;
    g_UpdState.FailRbkAGIndex = 0;
    g_UpdState.AtomGroupIndex = 0;
    g_UpdState.g_ClusInfoChange = 0;
    g_UpdState.g_MasterIp = "";
    g_UpdState.g_SwapMasterFalg = 0;
    g_UpdState.g_ClusterType = 0;
    g_UpdState.g_MaxBatchNo  = 0;
    g_UpdState.g_FinishAllBatch  = 0;
    g_UpdState.g_GetNidType = 0;
    g_UpdState.g_bHandleFault = 0;
    local func = loadstring("return " .. ParamList); --将字符串转换成table
    g_UpdState.ParamList = func();
    g_UpdState.ParamList.Flow = Flow;
    g_UpdState.NodeList = {};
    g_UpdState.CurNodeList = {};
    g_UpdState.SuccNodeList = {};
    g_UpdState.FailNodeList = {};
    g_UpdState.RbkNodeList = {};
    g_UpdState.MasterList = {};
    g_UpdState.g_CfgBatchNode={};
    g_UpdState.g_abNormalRebootFlag = 0;
    g_UpdState.g_AGPauseFalg = 0;
    g_UpdState.g_CurNodeNumForPause = 0;

    if("OnlineUpgrade" == g_UpdState.Flow)then
        InitClusterTypeByCfg();
    end

    g_UpdState.g_FlowStage = {};
    g_UpdState.g_isCrossUpgrade = 0;
    if (Flow == "OnlineUpgrade" or Flow == "OfflineUpgrade") then
        LogText(LL_NOTICE, "Enable the three stage flow.");
        g_UpdState.g_FlowStage.stage = STAGE_INIT_FLOW;
        g_UpdState.g_FlowStage.batch = STAGE_INIT_BATCH;
    else
        LogText(LL_NOTICE, "Disable the three stage flow.");
        g_UpdState.g_FlowStage.stage = STAGE_NOT_ENABLE;
        g_UpdState.g_FlowStage.batch = -1;
    end

    --初始化g_UpdState.NodeList
    func = loadstring("return " .. NodeList); --将字符串转换成table
    local tmp_node_list = func();
    for _, Node in pairs(tmp_node_list) do
        g_UpdState.NodeList[ip2int(Node.ip)] = {ip=Node.ip,tp="Manager",
            st=NODE_STATE_INIT,rt=ATOM_GROUP_EXEC_RET_INIT,bt=-1,et=-1,pr=0,batchno=0,ag=""};
    end

    --repair时恢复SPC版本号
    RecoverySpcVersionWhenRepair();

    --根据新文件state_version.lua恢复升级的版本信息
    ReusmeVersionInfo();
    for _, Node in pairs(g_UpdState.g_MasterSlave) do
        if(Node.IsMaster == "1") then
            g_UpdState.g_MasterIp = Node.ip;
            g_UpdState.MasterList[ip2int(Node.ip)] = {ip=Node.ip,type="Manager",
            state=ATOM_GROUP_EXEC_RET_INIT,rt=ATOM_GROUP_EXEC_RET_INIT,bt=-1,et=-1,pr=0,batchno=0,ag=""};
        end
    end

    if("OnlineUpgrade" == g_UpdState.Flow)then
        if(0 == g_UpdState.g_ClusterType)then
            if(GetSizeOfTable(g_UpdState.NodeList) > 2)then
                g_UpdState.g_ClusterType = BATCH_BY_DEFAULT;
                g_UpdState.g_MaxBatchNo  = 2;
            --双控还是使用老板使用的标记g_ClusterType为2表示双控,使用老代码获取批次
            else
                g_UpdState.g_ClusterType = 2;
                g_UpdState.g_MaxBatchNo  = 2;
            end
        end
    end
    SaveState(g_UpdState);
end

--------------------------------------------------------------------------------
-- Func Name       :InitPauseAGInfo
-- Description     :初始化AG暂停信息
--------------------------------------------------------------------------------
function InitPauseAGInfo()
    LogText(LL_NOTICE, "Exec InitPauseAGInfo.");
    g_UpdState.AutoSuspendAtomGroupInfo = g_UpdState.ParamList["pauseInfo"];
    if (nil == g_UpdState.AutoSuspendAtomGroupInfo) then
        LogText(LL_NOTICE, "The g_UpdState.AutoSuspendAtomGroupInfo is nil, No need init AG Pause information.");
        return;
    elseif (0 == string.len(g_UpdState.AutoSuspendAtomGroupInfo)) then
        LogText(LL_NOTICE, "Nothing in g_UpdState.AutoSuspendAtomGroupInfo, No need init AG Pause information.");
        return;
    end

    --在线升级开始，保存暂停信息
    --初始化暂停信息table
    LogText(LL_NOTICE, "Init some state for pause.");
    g_UpdState.AGPauseInfoArr = {};
    g_UpdState.g_AGPauseFalg = 0;
    g_UpdState.g_CurNodeNumForPause = 0;

    local PauseAGTable = SplitStr(g_UpdState.AutoSuspendAtomGroupInfo, ",");
    for index=1, #PauseAGTable do
        --记录暂停AG
        g_UpdState.AGPauseInfoArr[PauseAGTable[index]] = true;
    end
    LogText(LL_NOTICE, "Init AG pause information successed.");
    SaveState(g_UpdState);
    return;
end

function TranslateErrcode(ret)

    if (ret == EPERM) then
        --通知系统控制失败
        return "NotifySysStartUpgrade";
    elseif (ret == EBADF) then
        --创建升级标志文件失败
        return "CreateUpdStateFile";
    else
        return nil;
    end
end
--------------------------------------------------------------------------------
--启动一个流程
--Flow: string  流程名称
--NodeList: string 待升级节点列表
--ParamList: string 参数列表
--------------------------------------------------------------------------------
function Start(Flow, NodeList, ParamList)

    --启动新流程时递增FlowId
    local FlowId = g_UpdState.FlowId + 1;
    if (FlowId == 0) then
        FlowId = FlowId + 1;
    end
    
    --repair时备份SPC版本号
    BackupSpcVesionWhenRepair();
    
    --初始化流程执行信息
    InitExecFlowInfo(FlowId, Flow, NodeList, ParamList);
    LogText(LL_NOTICE, "Start flow: " .. g_UpdState.Flow .. " with flow id: " .. g_UpdState.FlowId);
       
    --初始化AG暂停信息
    InitPauseAGInfo();

    if (g_UpdState.g_bHandleFault == 1) then
        FinishFlowHandle();
        return FLOW_DISPATCH_STATE_FINISH;
    end

    --开始执行flow的预处理
    local ret = PreHandleBeforeUpdStart(Flow);
    if (RET_OK ~= ret) then
        LogText(LL_NOTICE, "Prehandle failed, terminate flow: " .. g_UpdState.Flow .. " with flow id: " .. g_UpdState.FlowId);
        --此时由于没有failNode，导致FinishFlowHandle中备份升级结果为成功，需要手动写入failNode
        for idx, CurNode in pairs(g_UpdState.MasterList) do
            table.insert(g_UpdState.FailNodeList, {ip = CurNode.ip, type = CurNode.tp, state = ATOM_GROUP_EXEC_RET_FAIL});
        end
        ReportAbnormalAGInfo{atom_group=TranslateErrcode(ret), node_ip=GetMasterIp(), error_code=3, result=1};
        FinishFlowHandle();
        return FLOW_DISPATCH_STATE_FINISH;
    end

    g_UpdState.FlowState = FLOW_STATE_RUNNING;
    return g_UpdateFlowsTbl[g_UpdState.Flow].func();
end


--------------------------------------------------------------------------------
-- Func Name       :UpdateRbkState
-- Description     :更新回退流程或者升级失败后回退流程的节点状态
--------------------------------------------------------------------------------
function UpdateRbkState(NodeIp, AtomGroup, Result)
    local info = g_UpdateFlowsTbl[g_UpdState.Flow].info;
    local progress = string.match(info, AtomGroup .. "=(%d+)") or 0;

    LogText(LL_NOTICE, "Get progress when rbk, atom group: " .. AtomGroup .. " progress: " .. progress);

    -- 更新NodeList状态
    local NodeT = g_UpdState.NodeList[ip2int(NodeIp)]
    if (NodeT ~= nil ) then
        NodeT.rt = Result
    end

    --更新本批次的成功节点列表和失败节点列表
    for Index, CurNode in pairs(g_UpdState.CurNodeList) do----AtomGroupName
        if (CurNode.ip == NodeIp) then
            --更新当前节点列表中的节点状态
            g_UpdState.CurNodeList[Index].state = Result;
            --回退成功时更新进度
            if (Result == ATOM_GROUP_EXEC_RET_SUCC_RBK) then
                if (progress - g_UpdState.NodeList[ip2int(CurNode.ip)].pr > 0) then
                    g_UpdState.NodeList[ip2int(CurNode.ip)].pr = progress;
                end
            end

            --检查该节点是否已经在失败节点列表中
            local InFailList = false;
            for FailIndex, FailNode in pairs(g_UpdState.FailNodeList) do
                if (FailNode.ip == NodeIp) then
                    InFailList = true;
                    break;
                end
            end

            --检查该节点是否已经在成功节点列表中
            local InSuccList = false;
            local SuccIndex = 0;
            for SuIndex, SuccNode in pairs(g_UpdState.SuccNodeList) do
                if (SuccNode.ip == NodeIp) then
                    InSuccList = true;
                    SuccIndex = SuIndex;
                    break;
                end
            end

            --回退成功的节点列表放入成功节点列表中
            if (Result == 3) then
                if (not InFailList) and (not InSuccList) then
                    table.insert(g_UpdState.SuccNodeList, {ip = CurNode.ip, type = CurNode.tp, state = Result});
                end
        --上报5标示热补丁回退、删除过程中发生异常重启
            elseif (5 == Result) then
                break;
            else --回退失败的节点放入失败节点列表中
                if not InFailList then
                    --从成功节点列表中删除该节点
                    if InSuccList then
                        table.remove(g_UpdState.SuccNodeList, SuccIndex);
                    end

                    table.insert(g_UpdState.FailNodeList, {ip = CurNode.ip, type = CurNode.tp, state = Result});
                end
            end

            SaveState(g_UpdState);
            break;
        end
    end
end

--------------------------------------------------------------------------------
-- Func Name       :IsNotInCurNodeList
-- Description     :不在当前节点列表中的节点
--------------------------------------------------------------------------------
function IsNotInCurNodeList(ip)
    for _, Node in pairs(g_UpdState.CurNodeList) do
        if(Node.ip == ip)then
            return false;
        end
    end
    return true;
end


--------------------------------------------------------------------------------
-- Func Name       : UpdateState
-- Description     : 更新升级流程执行时节点状态
--------------------------------------------------------------------------------
function UpdateState(NodeIp, AtomGroup, Result)
    local info = g_UpdateFlowsTbl[g_UpdState.Flow].info;
    local progress = string.match(info, AtomGroup .. "=(%d+)") or 0;

    LogText(LL_NOTICE, "Get progress, atom group: " .. AtomGroup .. " progress: " .. progress);

    -- 更新NodeList状态
    local NodeT = g_UpdState.NodeList[ip2int(NodeIp)]
    if (NodeT ~= nil ) then
        NodeT.rt = Result
    end

    --热补丁安装时，其他批次节点复位，加入失败节点列表
    if(IsNotInCurNodeList(NodeIp) and ((g_UpdState.Flow == "HotPatchUpgrade") or (g_UpdState.Flow == "AslPatchUpgrade")))then
        --检查该节点是否已经在失败节点列表中
        LogText(LL_NOTICE, "HotPatchUpgrade, atom group: " .. AtomGroup .. " NodeIp: " .. NodeIp .. " Result: " ..Result);
        local InFailList = false;
        for FailIndex, FailNode in pairs(g_UpdState.FailNodeList) do
            if (FailNode.ip == NodeIp) then
                InFailList = true;
                LogText(LL_NOTICE, "HotPatchUpgrade, node in FailList: " .. " NodeIp: " .. NodeIp);
                break;
            end
        end
        if not InFailList then
            --从成功节点列表中删除该节点
            for SuccIndex, SuccNode in pairs(g_UpdState.SuccNodeList) do
                if (SuccNode.ip == NodeIp) then
                    table.remove(g_UpdState.SuccNodeList, SuccIndex);
                    break;
                end
            end
            LogText(LL_NOTICE, "Insert, node in FailList: " .. " NodeIp: " .. NodeIp);
            table.insert(g_UpdState.FailNodeList, {ip = NodeIp, type = "Manager", state = Result});
        end
        SaveState(g_UpdState);
        return ;
    end
    ----升级PCIE交换机完成后，更新节点状态时，直接更新所有节点的状态;执行内部回退时，只需要更新根节点的状态，其他节点的状态不改变
    for Index, CurNode in pairs(g_UpdState.CurNodeList) do
        if (CurNode.ip == NodeIp) then
            --更新当前节点列表中的节点状态
            g_UpdState.CurNodeList[Index].state = Result;
            --升级成功时更新进度
            if (Result == ATOM_GROUP_EXEC_RET_SUCC) then
                --如果只升级PCIE交换机则进度置为100
                if (progress - g_UpdState.NodeList[ip2int(CurNode.ip)].pr > 0) then
                    g_UpdState.NodeList[ip2int(CurNode.ip)].pr = progress;
                end
            end
        end
    end

    --更新回退节点列表中的节点状态
    for Index, Node in pairs(g_UpdState.RbkNodeList) do
        if (NodeIp == Node.ip) then
            --更新回退节点列表中的节点状态
            g_UpdState.RbkNodeList[Index].state = Result;

            --回退成功时更新进度
            if (Result == ATOM_GROUP_EXEC_RET_SUCC_RBK) then
                if (progress - g_UpdState.NodeList[ip2int(Node.ip)].pr > 0) then
                    g_UpdState.NodeList[ip2int(Node.ip)].pr = progress;
                end
            end

            return;
        end
    end

    --更新本批次的成功节点列表和失败节点列表
    for Index, CurNode in pairs(g_UpdState.CurNodeList) do
        if (CurNode.ip == NodeIp) then
            --把升级失败的节点放入回退节点列表中--升级原子配置的结果，升级失败(FailExit)及回退成功(SuccRbk)都会加入回退列表
            if (ATOM_GROUP_EXEC_RET_FAIL == Result) or (ATOM_GROUP_EXEC_RET_TIMEOUT == Result) then
                local TmpNode = {};
                TmpNode.ip = CurNode.ip;
                TmpNode.tp = CurNode.tp;
                TmpNode.state = Result;
                TmpNode.index = 0;
                table.insert(g_UpdState.RbkNodeList, TmpNode);
            elseif (ATOM_GROUP_EXEC_RET_SUCC_RBK == Result) then--回退成功节点会查找回退的原子组
                local TmpNode = {};
                TmpNode.ip = CurNode.ip;
                TmpNode.tp = CurNode.tp;
                TmpNode.state = Result;
                TmpNode.index = 0;

                --查找回退AtomGroup XVE 查找回退原子不需要区分节点类型
                local RbkAtomGroupName = "";
                LogText(LL_NOTICE, "rollback atomname");
                RbkAtomGroupName = g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.AtomGroupIndex].rbk;

                if (nil == RbkAtomGroupName or RbkAtomGroupName == "") then
                    --没有对应的回退AtomGroup, 则状态设置为回退成功
                    TmpNode.state = 3;
                    LogText(LL_NOTICE, "Node (" .. CurNode.ip .. ") rollback finished.");
                end
                --TODO:内部回退已经删除，此处可以删除
                --设置回退AtomGroupIndex
                for i = 1, #g_UpdateFlowsTbl[g_UpdState.Flow].rbk do
                    if (g_UpdateFlowsTbl[g_UpdState.Flow].rbk[i].AtomGroupName == RbkAtomGroupName) then
                        TmpNode.index  = i;
                        break;
                    end
                end

                table.insert(g_UpdState.RbkNodeList, TmpNode);
            end

            if (ATOM_GROUP_EXEC_RET_SUCC == Result) then
                --检查该节点之前的AtomGroup是否执行失败
                local InFailList = false;
                for FailIndex, FailNode in pairs(g_UpdState.FailNodeList) do
                    if (FailNode.ip == NodeIp) then
                        InFailList = true;
                        break;
                    end
                end

                if not InFailList then
                    --检查该节点是否已经在成功节点列表中
                    local InSuccList = false;
                    for SuccIndex, SuccNode in pairs(g_UpdState.SuccNodeList) do
                        if (SuccNode.ip == NodeIp) then
                            InSuccList = true;
                            break;
                        end
                    end

                    if not InSuccList then
                        table.insert(g_UpdState.SuccNodeList, {ip = CurNode.ip, type = CurNode.tp, state = Result});
                    end
                end
            elseif (ATOM_GROUP_EXEC_RET_FAIL == Result) or (ATOM_GROUP_EXEC_RET_SUCC_RBK == Result) or (ATOM_GROUP_EXEC_RET_TIMEOUT == Result) then
                --检查该节点是否已经在失败节点列表中
                local InFailList = false;
                for FailIndex, FailNode in pairs(g_UpdState.FailNodeList) do
                    if (FailNode.ip == NodeIp) then
                        InFailList = true;
                        break;
                    end
                end

                if not InFailList then
                    --从成功节点列表中删除该节点
                    for SuccIndex, SuccNode in pairs(g_UpdState.SuccNodeList) do
                        if (SuccNode.ip == NodeIp) then
                            table.remove(g_UpdState.SuccNodeList, SuccIndex);
                            break;
                        end
                    end

                    table.insert(g_UpdState.FailNodeList, {ip = CurNode.ip, type = CurNode.tp, state = Result});
                end
            end

            SaveState(g_UpdState);
            break;
        end
    end
end

--------------------------------------------------------------------------------
-- Func Name       : GetAtomGroupEx
-- Description     : 获取节点当前执行的原子组信息
--------------------------------------------------------------------------------
function GetAtomGroupEx(NodeIp)
    LogText(LL_NOTICE, "GetAtomGroupEx NodeIp:" .. NodeIp);
    local Node = g_UpdState.NodeList[ip2int(NodeIp)]
    if (Node ~= nil) then
        return Node.ag;
    else
    ----此处主要针对恢复上电过程中,主控不在NodeList中的场景,其他场景采用第一个分支
        Node = g_UpdState.MasterList[ip2int(NodeIp)];
        LogText(LL_NOTICE, "Node is in MasterList." .. NodeIp);
        if (Node ~= nil) then
            return Node.ag;
        end
    end

    LogText(LL_NOTICE, "GetAtomGroupEx return nil.");
    return nil;
end


--------------------------------------------------------------------------------
-- Func Name       : GetAtomGroupState
-- Description     : 获取当前节点执行原子组的状态
--------------------------------------------------------------------------------
function GetAtomGroupState(NodeIp)

    local Node = g_UpdState.NodeList[ip2int(NodeIp)];
    if (Node ~= nil) then
        return Node.rt;
    else
        --此处主要针对恢复上电过程中,主控不在NodeList中的场景,其他场景采用第一个分支
        Node = g_UpdState.MasterList[ip2int(NodeIp)];
        if (Node ~= nil) then
            return Node.rt;
        end
    end

    LogText(LL_NOTICE, "GetAtomGroupState return nil.")
    return nil;
end

--------------------------------------------------------------------------------
-- Func Name       : IsAbnormalReboot
-- Description     : 当agent异常重启后，上报的原子组不是当前原子组，不判断重复消息
--如双控升级，主已经通知了AG到备节点执行，这时备发生复位且状态没有下盘，起来后节点报的AG还是上一个，此时不判重，继续触发下一个AG，否则主只能等待AG超时。
--------------------------------------------------------------------------------
function IsAbnormalReboot(AtomGroup, NodeIp)
    LogText(LL_NOTICE, "IsAbnormalReboot:" .. NodeIp .. "; " .. AtomGroup .. "; " .. g_UpdState.AtomGroupIndex .. "; " .. g_UpdState.Flow .. "; " .. g_UpdState.FlowUpdState)
    if ((1 == g_UpdState.FlowUpdState) and (("OnlineUpgrade" == g_UpdState.Flow) or ("Offlineupgrade" == g_UpdState.Flow))) then
        local Node = g_UpdState.NodeList[ip2int(NodeIp)]
        if ((nil ~= Node) and (nil ~= Node.ag) and (Node.ag ~= AtomGroup)) then
            LogText(LL_NOTICE, "The node exec abnormal reboot while rollbak and msg may wrong .")
            return true;
        end
    end
    if ( 0 == g_UpdState.AtomGroupIndex ) then
        LogText(LL_NOTICE, "g_UpdState.AtomGroupIndex is 0, so the msg is repeat ." )
        return false
    end
    if ( (0 == g_UpdState.FlowUpdState) and (("OnlineUpgrade" == g_UpdState.Flow) or ("OfflineUpgrade" == g_UpdState.Flow)) ) then
        if( (1 < g_UpdState.AtomGroupIndex) and (AtomGroup == g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.AtomGroupIndex - 1].AtomGroupName) ) then
            LogText(LL_NOTICE, "1.The node exec abnormal reboot and the msg may wrong ." )
            return true;
        end
        --主在线准备跳了AG的场景
        if( (2 < g_UpdState.AtomGroupIndex) and (AtomGroup == g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.AtomGroupIndex - 2].AtomGroupName) ) then
            LogText(LL_NOTICE, "2.The node exec abnormal reboot and the msg may wrong ." )
            return true;
        end
    end
    LogText(LL_NOTICE, "IsAbnormalReboot return false ." )
    return false
end

--------------------------------------------------------------------------------
-- Func Name       : IsRepeatAGMsg
-- Description     : 判断是否是重复消息
--------------------------------------------------------------------------------
function IsRepeatAGMsg(FlowId, AtomGroup, NodeIp)

    --若流程暂停，则不处理上报消息
    if (g_UpdState.FlowState == FLOW_STATE_UPD_SUSPEND or g_UpdState.FlowState == FLOW_STATE_RBK_SUSPEND or g_UpdState.FlowState == FLOW_STATE_END_SUSPEND) then
        LogText(LL_NOTICE, "Currently, flow state is suspended. It maybe a repeat message.");
        return TRUE;
    end

    -- 判断流水号是否相同
    LogText(LL_NOTICE, "Flow id and g_UpdState.FlowId:" .. FlowId .. "and" .. g_UpdState.FlowId)
    if (FlowId ~= g_UpdState.FlowId) then
        LogText(LL_NOTICE, "Flow id is different, maybe a repeat message.")
        return TRUE
    end

    -- 判断原子组名称是否相同，不相同说明是以前的消息
    local current = GetAtomGroupEx(NodeIp);
    if (current ~= nil) then
        LogText(LL_NOTICE, "current AtomGroup: "  .. current .. " and report AtomGroup: " .. AtomGroup);
    else
        LogText(LL_NOTICE, "current AtomGroup is nil and report AtomGroup: " .. AtomGroup);
    end
    --current == ""若满足此条件可能是节点还未执行升级时就发生了异常重启，此时上报的结果不处理
    if (current == nil or current == "" or AtomGroup ~= current) then
        if ( (false == IsNotInCurNodeList(NodeIp)) and (IsAbnormalReboot(AtomGroup, NodeIp))) then
            LogText(LL_NOTICE, "Master abnormal swap or node exec abnormal reboot, maybe save the wrong atomgroup.")
            return FALSE
        end
        LogText(LL_NOTICE, "Atom group name (" .. AtomGroup .. ") is different, maybe a repeat message.")
        return TRUE
    end

    -- 判断nodelist rt值是否为初值，不是则表明该消息已处理
    local state = GetAtomGroupState(NodeIp)
    if (nil ~= state) then
        LogText(LL_NOTICE, "state :" .. state)
    end
    if (state ~= ATOM_GROUP_EXEC_RET_INIT) then
        LogText(LL_NOTICE, "rt is not initial state, maybe a repeat message.")
        return TRUE
    end

    return FALSE

end

--------------------------------------------------------------------------------
-- Func Name       : AfterMangerRunAGUpdateState
-- Description     : 主控执行的AG完成后，更新节点的状态
--------------------------------------------------------------------------------
function AfterMangerRunAGUpdateState(AtomGroup, NodeIp, Result)
    LogText(LL_NOTICE, "-- Exce AfterMangerRunAGUpdateState--.");
    local index = 0;

    -- 同步PCIE原子组 无论执行成功失败流程都需要继续
    if (AtomGroup == "SyncPcieSwitch") then
        LogText(LL_NOTICE, "Current AtomGroup is SyncPcieSwitch.");
        Result = ATOM_GROUP_EXEC_RET_SUCC;
    end

    -- 升级PCIE执行失败流程都需要继续执行回退
    if ((AtomGroup == "UpgradePcieSw") and ((ATOM_GROUP_EXEC_RET_FAIL == Result) or (ATOM_GROUP_EXEC_RET_TIMEOUT == Result))) then
        LogText(LL_NOTICE, "Current AtomGroup is UpgradePcieSw.");
        for _, Node in pairs(g_UpdState.MasterList) do
            Node.state = Result;
            Node.rt = Result;
            UpdateState(Node.ip, AtomGroup, Result);
        end
        SaveState(g_UpdState);
        return;
    end

    --回退成功也需要更新，否则会导致进入fault状态
    if ((ATOM_GROUP_EXEC_RET_FAIL == Result) or (ATOM_GROUP_EXEC_RET_TIMEOUT == Result) or (ATOM_GROUP_EXEC_RET_SUCC_RBK == Result)) then
        --更新CurList和NodeList节点结果
        for _, Node in pairs(g_UpdState.CurNodeList) do
            LogText(LL_NOTICE, "Set CurNodeList fail:" .. Node.ip);
            if((g_UpdState.FlowUpdState == 0) and (not g_UpdateFlowsTbl[g_UpdState.Flow].isRbk))then
                UpdateState(Node.ip, AtomGroup, Result);
            else
                UpdateRbkState(Node.ip, AtomGroup, Result);
            end
        end
        --更新MasterList节点结果
        for _, Node in pairs(g_UpdState.MasterList) do
            Node.state = Result;
            Node.rt = Result;
            -- 主控IP不在CurNodeList中 但又执行了某些AG的情况 需要更新结果
            for _, NodeTmp in pairs(g_UpdState.NodeList) do
                if (Node.ip == NodeTmp.ip) then
                    NodeTmp.rt = Result;
                end
            end
        end
        SaveState(g_UpdState);
        return;
    end

    --执行成功，更新MasterList中根节的状态，并更新根节点的进度信息，然后继续执行后续的流程
    for _, Node in pairs(g_UpdState.MasterList) do
        Node.state = Result;
        Node.rt = Result;
        -- 主控IP不在CurNodeList中 但又执行了某些AG的情况 需要更新进度
        for _, NodeTmp in pairs(g_UpdState.NodeList) do
            if (Node.ip == NodeTmp.ip) then
                local progress = string.match(g_UpdateFlowsTbl[g_UpdState.Flow].info, AtomGroup .. "=(%d+)") or 0;
                if (progress - g_UpdState.NodeList[ip2int(NodeTmp.ip)].pr > 0) then
                    g_UpdState.NodeList[ip2int(NodeTmp.ip)].pr = progress;
                end
            end
            NodeTmp.rt = Result;
        end
    end
end
--------------------------------------------------------------------------------
-- Func Name       : ProcessOvertimeAtomGroup
-- Description     : 原子组超时后，上报原子组结果
--------------------------------------------------------------------------------
function ProcessOvertimeAtomGroup(AtomGroup, NodeIp, Result)
    if (Result ~= ATOM_GROUP_EXEC_RET_TIMEOUT) then
        return
    end

    ReportOvertimeAtom{atom_group=AtomGroup, node_ip=NodeIp, error_code=4,result=1}

    local Node = g_UpdState.NodeList[ip2int(NodeIp)]
    if (Node == nil ) then
        return
    end

    LogText(LL_NOTICE, "ProcessOvertimeAtomGroup: --AGName: " .. AtomGroup .. "--ip: " .. NodeIp .. "--result: " .. Result);
end

--------------------------------------------------------------------------------
-- Func Name       : Run
-- Description     : 节点上报执行结果，继续运行当前流程
-- Caution         : 由升级框架调用，函数名称、参数、返回值必须与模板一致
-- Input           : FlowId: int 节点当前执行的流程的流水号
--                   AtomGroup: string 节点当前执行的原子组名称
--                   NodeIp: string 上报执行结果的节点IP地址
--                   Result: int 节点执行原子组的结果
--                           ATOM_GROUP_EXEC_RET_INIT:未执行
--                           ATOM_GROUP_EXEC_RET_SUCC: 执行成功
--                           ATOM_GROUP_EXEC_RET_FAIL:执行失败回退失败
--                           ATOM_GROUP_EXEC_RET_SUCC_RBK:执行失败回退成功
--                           ATOM_GROUP_EXEC_RET_TIMEOUT: 执行超时
--                           ATOM_GROUP_EXEC_RET_PAUSE: 暂停
-- Output          :
-- Return          : 向节点分发任务状态
--                   FLOW_DISPATCH_STATE_NORMAL : 正常
--                   FLOW_DISPATCH_STATE_FINISH : 当前流程已经执行完成
--                   FLOW_DISPATCH_STATE_NO_FLOW: 当前没有流程正在执行
--                   FLOW_DISPATCH_STATE_WRONG_IP: 节点IP地址错误
--------------------------------------------------------------------------------
function Run(FlowId, AtomGroup, NodeIp, Result)
    LogText(LL_NOTICE, "Run flow: " .. g_UpdState.Flow .. "--AGName: " .. AtomGroup .. "--ip: " .. NodeIp .. "--result: " .. Result);

    if ((nil == g_UpdState) or (nil == g_UpdState.Flow) or ("" == g_UpdState.Flow)) then
        LogText(LL_WARN, "No flow is running.");
        return FLOW_DISPATCH_STATE_NO_FLOW;
    end

    local Cmd = "/OSM/script/os_system_update.sh -O";
    local Ret = System(Cmd, 1);
    if(RET_OK ~= Ret)then
    -- 跨版本升级 SR-0000469170 V300R003升级路径支持 20150306
    if ((g_UpdState.ParamList.rollversion ~= nil)
        and (true ~= file_exists("/startup_disk/conf/conf_local/state_version.lua"))
        and (string.find(g_UpdState.ParamList.rollversion, "^32001") or string.find(g_UpdState.ParamList.rollversion, "^31801"))) then
        LogText(LL_NOTICE, "Run upgrade from R version: " .. g_UpdState.ParamList.rollversion);
        g_UpdState.g_FlowStage = {stage = -1, batch = -1, agindex = -1};
        if ((g_UpdState.Flow == "OnlineUpgrade") or (g_UpdState.Flow == "OfflineUpgrade")) then
            g_UpdState.g_FlowStage.stage = STAGE_SECOND_FLOW;
            g_UpdState.g_FlowStage.batch = STAGE_INIT_BATCH;
            g_UpdState.g_isCrossUpgrade = 1;
        end
    end
    end
    if (AtomGroup == nil or AtomGroup == "" or NodeIp == nil or NodeIp == "") then
        LogText(LL_ERROR, "Node IP is nil.");
        return FLOW_DISPATCH_STATE_WRONG_IP;
    end

    if (Result == nil or Result == "") then
        Result = ATOM_GROUP_EXEC_RET_FAIL;
    end

    if (g_UpdState.FlowState == FLOW_STATE_FINISH) then
        LogText(LL_WARN, "Flow finished.");
        UpdateNodeState(TERM_FLOW, {}, nil);
        PostHandleAfterUpdFinish(g_UpdState.Flow);
        return FLOW_DISPATCH_STATE_FINISH;
    end

    if (IsAbnormalReboot(AtomGroup, NodeIp) and (Result == ATOM_GROUP_EXEC_RET_SUCC or ATOM_GROUP_EXEC_RET_SUCC_RBK == Result)) then
        LogText(LL_WARN, "Abnormal reboot and change Result to 2.");
        Result = ATOM_GROUP_EXEC_RET_FAIL
        if (1 == g_UpdState.FlowUpdState) then
            g_UpdState.g_abNormalRebootFlag = 1;
            LogText(LL_WARN, "The node exec abnormal reboot while rollbak and set abnormal reboot flag 1.");
        end
    end

     -- 上报超时原子组
    ProcessOvertimeAtomGroup(AtomGroup, NodeIp, Result)

    -- 仅在主控执行的原子组
    if ((g_UpdState.g_FlowStage.stage ~= STAGE_FIRST_FLOW) and (g_UpdState.g_FlowStage.stage ~= STAGE_THIRD_FLOW)) then
        if ((g_UpdState.FlowUpdState == 0 and "Manager" == g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.AtomGroupIndex].ExcuteNodeType)
           or (g_UpdState.FlowUpdState == 1 and "Manager" == g_UpdateFlowsTbl[g_UpdState.Flow].rbk[g_UpdState.AtomGroupIndex].ExcuteNodeType)) then
            AfterMangerRunAGUpdateState(AtomGroup, NodeIp, Result);
            return g_UpdateFlowsTbl[g_UpdState.Flow].func();
        end
    end

    LogText(LL_NOTICE, "---g_UpdState.FlowUpdState----" .. g_UpdState.FlowUpdState);
    if ((not g_UpdateFlowsTbl[g_UpdState.Flow].isRbk) and (g_UpdState.FlowUpdState == 0))then
        --执行升级流程时更新节点状态
        UpdateState(NodeIp, AtomGroup, Result);
    else
        --执行回退流程及升级失败回退时更新节点状态
        UpdateRbkState(NodeIp, AtomGroup, Result);
    end

    if ("CopyPackage" == AtomGroup) then
        RunCopyPackageAG(NodeIp, Result)
    end

    --检查当前批次的节点是否执行完成, 未执行完成则直接返回，等待节点执行完成
    for _, Node in pairs(g_UpdState.CurNodeList) do
        if (Node.state == ATOM_GROUP_EXEC_RET_INIT) then
            LogText(LL_ERROR, "Node not finished: " .. Node.ip);
            return FLOW_DISPATCH_STATE_NORMAL;
        end
    end

    PreRunFlowHandle();
    return g_UpdateFlowsTbl[g_UpdState.Flow].func();
end

function GetFailNodeNid(NodeList)
    LogText(LL_NOTICE, "Exec Get Fail Node Nid." );
    local FailNodeList = "";
    local TmpNid;

    for _, FailNode in pairs(g_UpdState.FailNodeList) do
        for _, Node in pairs(NodeList) do
            if (FailNode.ip == Node.ip) then
                TmpNid = GetNidByIp(FailNode.ip);
                if(nil ~= TmpNid)then
                    FailNodeList = FailNodeList .. TmpNid .. "," ;
                end
                break;
            end
        end
    end

    LogText(LL_NOTICE, "Fail Node List: ", FailNodeList);
    return FailNodeList;
end

function BuildParamForRetryAG(AtomGroup)
    LogText(LL_NOTICE, "Build Param For Retry AG : " .. AtomGroup.AtomGroupName);

    --将集群节点个数传到upd_atom.lua侧，用于前端链路冗余状态检查
    local Ret = 0;
    local NodeNum = 0;
    Ret, NodeNum = GetUsrConfigNodeNum();
    if(RET_OK == Ret) then
        g_UpdState.ParamList.NodeTotalNum = NodeNum;
    else
        g_UpdState.ParamList.NodeTotalNum = #(g_UpdState.g_ClusterInfo);
    end

    --最后一批检查多路径失败不需要回退
    if ("SysCompatibleAfterReboot" == AtomGroup.AtomGroupName)then
        local Ret = 0;
        local NodeNum = 0;
        Ret, NodeNum = GetUsrConfigNodeNum();
        g_UpdState.ParamList.RollBackFlag = 1;
        if(RET_OK == Ret)then
            if((1 == g_UpdState.g_FinishAllBatch) or ((2 == g_UpdState.BatchNo) and (NodeNum <= 2)))then
                g_UpdState.ParamList.RollBackFlag = 0;
            end
        end
    end

    if("ALL" == AtomGroup.ExcuteNodeType)then
        if("OnlineUpgrade" == g_UpdState.Flow)then
            local nidlist;
            nidlist = GetFailNodeNid(g_UpdState.CurNodeList);
            LogText(LL_NOTICE, "GetFailNodeNid nidlist: ".. nidlist);
            g_UpdState.ParamList.nidlist = nidlist;
        else
            local nidlist;
            nidlist = GetFailNodeNid(g_UpdState.NodeList);
            LogText(LL_NOTICE, "GetFailNodeNid nidlist: ".. nidlist);
            g_UpdState.ParamList.nidlist = nidlist;
        end
    end
end

function removeNodeFromFailAndRbkList(NodeIp)
    local index = nil;
    for idx, node in pairs(g_UpdState.FailNodeList) do
        if (node.ip == NodeIp) then
            index = idx;
            break;
        end
    end
    if (index ~= nil) then
        table.remove(g_UpdState.FailNodeList, index);
    end

    index = nil;
    for idx, node in pairs(g_UpdState.RbkNodeList) do
        if (node.ip == NodeIp) then
            index = idx;
            break;
        end
    end
    if (index ~= nil) then
        table.remove(g_UpdState.RbkNodeList, index);
    end
end

function addToSuccNodeList(NodeIp)
    for _, node in pairs(g_UpdState.SuccNodeList) do
        if (node.ip == NodeIp) then
            return;
        end
    end
    table.insert(g_UpdState.SuccNodeList, {ip = NodeIp, state = ATOM_GROUP_EXEC_RET_SUCC});
end

function notifyFailNodesExeAG(AtomGroup)
    if ("Manager" == AtomGroup.ExcuteNodeType) then
        for _, Node in pairs(g_UpdState.MasterList) do
            removeNodeFromFailAndRbkList(Node.ip);
        end
        for _, CurNode in pairs(g_UpdState.CurNodeList) do
            removeNodeFromFailAndRbkList(CurNode.ip);
            addToSuccNodeList(CurNode.ip);
            CurNode.state = ATOM_GROUP_EXEC_RET_SUCC;
        end
        ManagerRunAtomGroup(AtomGroup);
        return;
    end

    for _, CurNode in pairs(g_UpdState.CurNodeList) do
        if((g_UpdState.NodeList[ip2int(CurNode.ip)].ag == AtomGroup.AtomGroupName)
            and (CurNode.state ~= ATOM_GROUP_EXEC_RET_INIT ) and (CurNode.state ~= ATOM_GROUP_EXEC_RET_SUCC)) then
            CurNode.state = ATOM_GROUP_EXEC_RET_INIT;
            removeNodeFromFailAndRbkList(CurNode.ip);
            NodeExecAtomGroup(CurNode, AtomGroup, Table2String(g_UpdState.ParamList));
            os.execute("sleep " .. 10);
        end
    end
end

function GetRetryNodeList()
    AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.AtomGroupIndex];
    local retryNodeList="";
    local retryNid = nil;
    if ("Manager" == AtomGroup.ExcuteNodeType) then
        for _, Node in pairs(g_UpdState.MasterList) do
            retryNid = GetNidByIp(Node.ip);
            if (nil ~= retryNid) then
                retryNodeList = retryNodeList .. retryNid .. ",";
            end
        end
        return retryNodeList;
    end

    for _, CurNode in pairs(g_UpdState.CurNodeList) do
        if((g_UpdState.NodeList[ip2int(CurNode.ip)].ag == AtomGroup.AtomGroupName)
            and (CurNode.state ~= ATOM_GROUP_EXEC_RET_INIT ) and (CurNode.state ~= ATOM_GROUP_EXEC_RET_SUCC)) then
            retryNid = GetNidByIp(CurNode.ip);
            if (nil ~= retryNid) then
                retryNodeList = retryNodeList .. retryNid .. ",";
            end
        end
    end
    return retryNodeList;
end

function Retry()
    --将异常主备切换标记清除
    g_UpdState.g_SwapMasterFalg = 0;
    --恢复状态
    g_UpdState.FlowState = FLOW_STATE_RUNNING;

    -- 一三阶段专家模式适配
    if ((g_UpdState.g_FlowStage.stage == STAGE_FIRST_FLOW)
     or (g_UpdState.g_FlowStage.stage == STAGE_THIRD_FLOW)) then
        -- 全成功
        if (IsNodeFailInCurNodeList() == false) then
            return g_UpdateFlowsTbl[g_UpdState.Flow].func();
        end

        -- 重试哪个AG
        local AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].preupd[1];
        g_UpdState.ParamList.stage = STAGE_FIRST_FLOW;
        if(STAGE_THIRD_FLOW == g_UpdState.g_FlowStage.stage) then
            AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].postupd[1];
            g_UpdState.ParamList.stage = STAGE_THIRD_FLOW;
        end

        g_UpdState.ParamList.updType = GetUpdType();
        g_UpdState.ParamList.retry = "retry";

        -- 重试哪个节点
        for _, Node in pairs(g_UpdState.CurNodeList) do
            -- 初始态的AG用不用下发
            if ((Node.state ~= ATOM_GROUP_EXEC_RET_INIT) and (Node.state ~= ATOM_GROUP_EXEC_RET_SUCC)) then
                Node.state = ATOM_GROUP_EXEC_RET_INIT;
                removeNodeFromFailAndRbkList(Node.ip);
                NodeExecAtomGroup(Node, AtomGroup, Table2String(g_UpdState.ParamList));
            end
        end

        --清除重试标记
        g_UpdState.ParamList.retry = nil;
        SaveState(g_UpdState);
        return FLOW_DISPATCH_STATE_NORMAL;
    end

    --设置重试标记，提前设置是为了异常主备切换导致，流程暂停之后更换备件之后，可能导致了系统回退到老版本，需要全部重升级
    g_UpdState.ParamList.retry = "retry";

    --若当前批次节点当前原子组都执行成功了，则直接继续（比如异常主备切换导致的回退）
    if (IsAllNodeSucc()) then
        local iRet = g_UpdateFlowsTbl[g_UpdState.Flow].func();
        g_UpdState.ParamList.retry = nil;
        SaveState(g_UpdState);
        return iRet;
    end

    --传暂停标识到Agent执行atom
    g_UpdState.ParamList.pauseFlag = g_UpdState.g_AGPauseFalg;
    g_UpdState.g_AGPauseFalg = 0;

    --若当前批次节点当前原子组有节点失败了，则让这些节点重试一次该AG
    AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.AtomGroupIndex];

    --为重试原子组拼接参数
    BuildParamForRetryAG(AtomGroup);

    --通知失败节点重试执行AG
    notifyFailNodesExeAG(AtomGroup);

    --清除重试标记
    g_UpdState.ParamList.retry = nil;
    g_UpdState.ParamList.pauseFlag = nil;
    SaveState(g_UpdState);
    return FLOW_DISPATCH_STATE_NORMAL;
end

function Continue(param)
    local func = loadstring("return " .. param);
    local ConData = func();
    g_UpdState.ParamList.continue = ConData.continue;
    NotifyMmtWhenSuspend(false);
    if (MODE_SHIELD_ALARM == g_UpdState.ParamList.shieldalarmtype) then
        NotifyAlarmShield(true);                --专家模式恢复流程的时候屏蔽告警
    end
    NotifyDrvUpdState{upd_state = 1, upd_flowname = g_UpdState.Flow, upd_locationid = "all"};--专家模式恢复流程的时候关闭各节点例测
    if (ConData.continue == CON_TYPE_RETRY) then
        return Retry();
    elseif (ConData.continue == CON_TYPE_UPD or ConData.continue == CON_TYPE_ROLL) then
        return g_UpdateFlowsTbl[g_UpdState.Flow].func();
    elseif (ConData.continue == CON_TYPE_END) then
        LogText(LL_NOTICE, "End flow for professional mode.");
        g_UpdState.FlowState = FLOW_STATE_FINISH;
        FinishFlowHandle();
        SaveState(g_UpdState);
        return FLOW_DISPATCH_STATE_FINISH;
    else
        g_UpdState.FlowState = FLOW_STATE_FINISH;
        SaveState(g_UpdState);
        return FLOW_DISPATCH_STATE_FINISH;
    end
end
--------------------------------------------------------------------------------
-- Func Name       : Terminate
-- Description     : 终止当前正在执行的升级流程
-- Caution         : 由升级框架调用，函数名称、参数、返回值必须与模板一致
-- Input           : FlowId: int 流程的序列号
-- Output          :
-- Return          :
--------------------------------------------------------------------------------
function Terminate(FlowId)
    if (g_UpdState.FlowId ~= FlowId) then
        LogText(LL_ERROR, "Terminate flow failed: " .. FlowId);
        return;
    end

    for _, Node in pairs(g_UpdState.NodeList) do
        --atom名字赋值成Flow名字，此种情况上报FLOW超时
        ReportOvertimeAtom{atom_group=GetAtomGroupEx(Node.ip), atom=g_UpdState.Flow, node_ip=Node.ip, error_code=4,result=1}
    end

    UpdateNodeState(TERM_FLOW, {}, nil);
    g_UpdState.FlowState = FLOW_STATE_FINISH;
    SaveState(g_UpdState);
    PostHandleAfterUpdFinish(g_UpdState.Flow);
    LogText(LL_WARN, "Terminate flow: " .. FlowId);
end

function AgIndex2AgName(AgIndex)
    return g_UpdateFlowsTbl[g_UpdState.Flow].upd[AgIndex].AtomGroupName;
end

function AgExecFinish(Result)
    if (Result == 0) or (Result == 2) or (Result == 3) or (Result == 4) then
        return true;
    end

    return false;
end

function VerifyAgentStateInfo(FlowIdIsSame, AgentIndex, AgentResult)
    if (FlowIdIsSame == 0) and (AgExecFinish(AgentResult)) and (AgIndex2AgName(AgentIndex) == "OfflineSysCompatibleAfterReboot") then
        LogText(LL_NOTICE, "Verify state return true.");
        return true;
    end

    LogText(LL_NOTICE, "Verify state return false.");
    return false;
end

function AnalyseAndReport(NodeIp)
    local agentState = "/startup_disk/conf/conf_local/upd_state_atoms.lua";
    local serverState = "/startup_disk/conf/conf_local/upd_state_flows.lua";
    local tmpDir = "/tmp/upd_state_atoms";
    --清空临时目录
    local cmd = "[ -d /tmp/upd_state_atoms ] && { rm -rf /tmp/upd_state_atoms/*; } || { mkdir -p /tmp/upd_state_atoms; }";
    local ret = System(cmd, 2);
    if (ret ~= 0) then
        LogText(LL_NOTICE, "Failed to clear dir /tmp/upd_state_atoms for ip:" .. NodeIp);
        return;
    end
    --如果待分析节点是本控，即主控，则agent状态文件从本控拷贝；否则从对应节点GET过来
    if (IsMasterNode(NodeIp)) then
        cmd = "cp -f " .. agentState .. " " .. tmpDir;
        ret = System(cmd, 2);
    else
        ret = TransSomethingByRealIP{virtual_ip = NodeIp, trans_type = "GET_FILE", src = agentState, dst = tmpDir, timeout=5};
    end
    if (ret ~= 0) then
        LogText(LL_NOTICE, "Failed to get agent state file for ip:" .. NodeIp);
        return;
    end
    --从本控拷贝server状态文件
    cmd = "cp -f " .. serverState .. " " .. tmpDir;
    ret = System(cmd, 2);
    if (ret ~= 0) then
        LogText(LL_NOTICE, "Failed to copy server state file for ip:" .. NodeIp);
        return;
    end
    --分析该节点是否需要模拟上报
    cmd = "/OSM/script/os_system_update.sh -H flowid";
    local FlowIdIsSame = System(cmd, 2);

    cmd = "/OSM/script/os_system_update.sh -H agindex";
    local AgentAgIndex = System(cmd, 2);

    cmd = "/OSM/script/os_system_update.sh -H result";
    local AgentResult = System(cmd, 2);

    LogText(LL_NOTICE, "Analyse state, FlowIdIsSame: " .. FlowIdIsSame .. " AgIndex: " .. AgentAgIndex .. " AgName: " .. AgIndex2AgName(AgentAgIndex) .. " AgentResult: " .. AgentResult);

    if (VerifyAgentStateInfo(FlowIdIsSame, AgentAgIndex, AgentResult)) then
        ReportAtomGroup{flow_id=g_UpdState.FlowId, atom_group=AgIndex2AgName(AgentAgIndex), node_ip=NodeIp, result=AgentResult};
    end
end

function ReportAgentStateWhenResume()
    LogText(LL_NOTICE, "Report state when resume, FlowName: " .. g_UpdState.Flow .. " FlowUpdState: " .. g_UpdState.FlowUpdState .. " g_SwapMasterFalg: " .. g_UpdState.g_SwapMasterFalg);

    if ("OfflineUpgrade" == g_UpdState.Flow) and (0 == g_UpdState.FlowUpdState) and (1 == g_UpdState.g_SwapMasterFalg) then
        for _, Node in pairs(g_UpdState.CurNodeList) do
            local NodeTab = g_UpdState.NodeList[ip2int(Node.ip)];
            if (nil ~= NodeTab) then
                LogText(LL_NOTICE, "Report state when resume, Node.state: " .. Node.state .. " NodeTab.ag: " .. NodeTab.ag .. " Node.ip: " .. Node.ip);
                if (ATOM_GROUP_EXEC_RET_INIT == Node.state) and (NodeTab.ag == "OfflineSysCompatibleAfterReboot") then
                    AnalyseAndReport(Node.ip);
                end
            end
        end
    end
end
function UpdateCurFinishStatus()
    local curBatch = 0;
    local batchno = 0;
    local initnode = 0;
    for _, Node in pairs(g_UpdState.NodeList) do
        batchno = g_UpdState.NodeList[ip2int(Node.ip)].batchno ;
        --查找未升级的节点
        if(0 == batchno) then
            initnode = initnode + 1;
        end
        --当前升级节点的最大批次信息
        if(curBatch < batchno)then
            curBatch = batchno
        end
    end
    --batchno为初始值0的节点个数，若所有节点的批次信息都不是0则表示已经升级到最后一个批次
    --g_ClusterType在新版本获取批次时使用,双控在新版本判断异常时会用
    --新版本的g_ClusterType不需要重新赋值,g_MaxBatchNo可以确定所以直接赋值
    if(0 == initnode)then
        g_UpdState.g_FinishAllBatch = 1;
        g_UpdState.g_MaxBatchNo = curBatch;
        if(nil == g_UpdState.g_ClusterType)then
            if(((2 == curBatch) and (GetSizeOfTable(g_UpdState.NodeList) == 2))
                or ((3 == curBatch) and (GetSizeOfTable(g_UpdState.NodeList) > 2)))then
                g_UpdState.g_ClusterType = curBatch;
                LogText(LL_NOTICE, "Online resume info:g_UpdState.g_ClusterType is nil and resume cluster type:" .. g_UpdState.g_ClusterType);
            end
        end
    else
        --新版本只有最后一批才会使用g_ClusterType,g_MaxBatchNo非最后一批不需要处理这些值
        --新版本主上恢复的值为0,表示还有节点未升级完成,后继续升级时会将其再更新成1
        --finishflow调用后将新增的全局变量g_MaxBatchNo置空
        g_UpdState.g_FinishAllBatch = 0;
        --新版本不需要重新设置变量值
        if(0 == g_UpdState.g_MaxBatchNo or nil == g_UpdState.g_MaxBatchNo)then
            if(2 == GetSizeOfTable(g_UpdState.NodeList))then
                g_UpdState.g_MaxBatchNo = 2;
                g_UpdState.g_ClusterType = 2;
            else
                g_UpdState.g_MaxBatchNo = 3;
                if(nil == g_UpdState.g_ClusterType)then
                    g_UpdState.g_ClusterType = 3;
                    LogText(LL_NOTICE, "Online resume info:g_UpdState.g_ClusterType is nil and resume cluster type:" .. g_UpdState.g_ClusterType);
                end
            end
        end
        --多次发生主备倒换后在老版本结束升级流程，残留了状态文件后续双控变成四控继续升级
        if(2 == g_UpdState.g_ClusterType or 3 == g_UpdState.g_ClusterType)then
            g_UpdState.g_MaxBatchNo = g_UpdState.g_ClusterType;
        end
    end
    LogText(LL_NOTICE, "Online resume info:g_FinishAllBatch." .. g_UpdState.g_FinishAllBatch);
end
function PreRunFlowHandle()
    --在线升级兼容老版本新版本新增变量:g_ClusterType, g_MaxBatchNo, g_FinishAllBatch
    if("OnlineUpgrade" == g_UpdState.Flow)then
        --g_UpdState.g_FinishAllBatch为nil为跨版本升级可以直接赋值
        if(nil == g_UpdState.g_FinishAllBatch) then
            g_UpdState.g_FinishAllBatch = 0;
            if(GetSizeOfTable(g_UpdState.NodeList) > 2) then
                g_UpdState.g_ClusterType = 3;
                g_UpdState.g_MaxBatchNo = 3;
                if(g_UpdState.BatchNo == g_UpdState.g_MaxBatchNo)then
                    g_UpdState.g_FinishAllBatch = 1;
                end
            else
                g_UpdState.g_ClusterType = 2;
                g_UpdState.g_MaxBatchNo = 2;
                if(g_UpdState.BatchNo == g_UpdState.g_MaxBatchNo)then
                    g_UpdState.g_FinishAllBatch = 1;
                end
            end
            LogText(LL_NOTICE, "Online resume info:g_MaxBatchNo,g_ClusterType,g_FinishAllBatch." .. g_UpdState.g_MaxBatchNo .. g_UpdState.g_ClusterType .. g_UpdState.g_FinishAllBatch);
        else
            --若状态为非nil值，则根据当前节点相关的批次信息设置相关的状态值
            UpdateCurFinishStatus();
        end
    end
end
function IsAllNodeFinish()
    for _, Node in pairs(g_UpdState.NodeList) do
        --节点状态为初始状态说明节点未执行 节点的状态为执行中说明节点正在执行升级操作
        if (Node.st == NODE_STATE_INIT or Node.st == NODE_STATE_RUNNING) then
            LogText(LL_NOTICE, "Some node do not finish upgrade.");
            return false;
        end
    end
    LogText(LL_NOTICE, "All node upgrade finish.");
    return true;
end

function NotifySysUpgradeFinsih()
    local flag = 0;
    
    if ((g_UpdState.FlowState == FLOW_STATE_RUNNING) and (g_UpdState.Flow == "OnlineUpgrade" or g_UpdState.Flow == "OfflineUpgrade")) then
        if(IsAllNodeFinish())then
            FinishFlowHandle();
            return FLOW_STATE_FINISH;
        end
    end
    
    --问题单DTS2017062211716，修改升级流程开始，第一个AG还没有上报时集群复位后，升级流程停止，查询状态显示升级中问题
    if ((g_UpdState.FlowState == FLOW_STATE_FINISH) and (g_UpdState.Flow == "OnlineUpgrade" or g_UpdState.Flow == "OfflineUpgrade")) then
        --升级状态为FINISH，所有节点执行状态都为init时，结束升级流程
        for _, Node in pairs(g_UpdState.NodeList) do
            if (Node.st ~= NODE_STATE_INIT) then
                LogText(LL_NOTICE, "Some node do not finish upgrade.");
                flag = 1;
            end
        end 
        if(0 == flag)then
            LogText(LL_WARN, "FlowState is FLOW_STATE_FINISH and Node.st is INIT, finish the flow.");
            FinishFlowHandle();
            return FLOW_STATE_FINISH;
            
        end      
    end
        
    --通知系统控制开始后，主发生异常，需要通知回退成功
    if (g_UpdState.g_FlowStage ~= nil and g_UpdState.g_FlowStage.stage == STAGE_INIT_FLOW and g_UpdState.FlowState == FLOW_STATE_FINISH) then
        local cmd = "/OSM/bin/upgrade_atom -f NOTIFY_SYS_UPDATE_STATE -u 3 -h " .. g_UpdState.Flow .. " -a 0";
        System(cmd, 31);
        LogText(LL_NOTICE, "Master reboot when start the flow, notify sys failed.");
        DeleteUpdStateAllowSync(g_UpdState.Flow);

    end
end

--------------------------------------------------------------------------------
-- Func Name       : RestoreAGTimer
-- Description     : 进程恢复时调用，如AG Timer未成功恢复，则通过此函数重新恢复
--                   相应的AG Timer
--------------------------------------------------------------------------------
function RestoreAGTimer()
    local AtomGroup =  nil;

    for _, Node in pairs(g_UpdState.CurNodeList) do
        -- 当前批次的节点还未执行完成
        AtomGroup = nil;
        if ((ATOM_GROUP_EXEC_RET_INIT == Node.state) and (nil ~= g_UpdState.NodeList[ip2int(Node.ip)])) then
            NodeTab = g_UpdState.NodeList[ip2int(Node.ip)];
            ---升级流程节点异常重启
            if (0 == g_UpdState.FlowUpdState) then
                for i = 1, #g_UpdateFlowsTbl[g_UpdState.Flow].upd do
                    if (g_UpdateFlowsTbl[g_UpdState.Flow].upd[i].AtomGroupName == NodeTab.ag) then
                        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[i];
                        break;
                    end
                end
            else
                ---回退流程节点异常重启
                for i = 1, #g_UpdateFlowsTbl[g_UpdState.Flow].rbk do
                    if (g_UpdateFlowsTbl[g_UpdState.Flow].rbk[i].AtomGroupName == NodeTab.ag) then
                        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].rbk[i];
                        break;
                    end
                end
            end

            if (nil ~= AtomGroup) then
                iRet = ExecRestoreAGTimer{atom_group=AtomGroup.AtomGroupName, node_ip=Node.ip, over_time=AtomGroup.timeout};
                if (RET_OK == iRet) then
                    LogText(LL_NOTICE, "restart node [" .. Node.ip .."]AG " .. AtomGroup.AtomGroupName .. "timer." .. AtomGroup.timeout);
                end
            end
        end
    end
end

--------------------------------------------------------------------------------
-- Func Name       : Resume
-- Description     : 进程恢复时调用，恢复流程的执行状态
--------------------------------------------------------------------------------
function Resume()
    if (nil == g_UpdState) or (nil == g_UpdState.Flow)
        or ("" == g_UpdState.Flow) or (nil == g_UpdState.FlowState) then
        LogText(LL_NOTICE, "No flow is executing.");
        g_UpdState.FlowState = FLOW_STATE_FINISH;
        SaveState(g_UpdState);
        return FLOW_STATE_FINISH;
    end

    local Cmd = "/OSM/script/os_system_update.sh -O";
    local Ret = System(Cmd, 1);
    if(RET_OK ~= Ret)then
    -- 跨版本升级
    if ((g_UpdState.ParamList.rollversion ~= nil)
        and (true ~= file_exists("/startup_disk/conf/conf_local/state_version.lua"))
        and (string.find(g_UpdState.ParamList.rollversion, "^32001") or string.find(g_UpdState.ParamList.rollversion, "^31801"))) then
        LogText(LL_NOTICE, "Resume upgrade from R version: " .. g_UpdState.ParamList.rollversion);
        g_UpdState.g_FlowStage = {stage = -1, batch = -1, agindex = -1};
        if ((g_UpdState.Flow == "OnlineUpgrade") or (g_UpdState.Flow == "OfflineUpgrade")) then
            g_UpdState.g_FlowStage.stage = STAGE_SECOND_FLOW;
            g_UpdState.g_FlowStage.batch = STAGE_INIT_BATCH;
            g_UpdState.g_isCrossUpgrade = 1;
        end
    end
    end
    LogText(LL_NOTICE, "Resume flow: " .. g_UpdState.Flow);
    if (("SyncUpgrade" == g_UpdState.Flow) or (nil ~= string.find(g_UpdState.Flow, "Insert"))) then
        ---保存流程执行状态为完成，当agent执行resume上报执行结果执行Run函数时，直接返回，避免导致升级状态改变。
        g_UpdState.FlowState = FLOW_STATE_FINISH;
        SaveState(g_UpdState);
        return FLOW_STATE_FINISH;
    end

    if (FLOW_STATE_FINISH ~= g_UpdState.FlowState) then
        if(1 == CreateUpgradeStateFile())then
            LogText(LL_WARN, "Fail to create the upd state file when resume.");
        end
    end

    PreRunFlowHandle();
    -- 如节点的AG timer未恢复，此处将重新恢复其AG Timer
    RestoreAGTimer();
    -- server侧resume时查询上报agent状态
    ReportAgentStateWhenResume();
    -- 判断是否在线离线升级的第二阶段并且所有节点执行完成
    NotifySysUpgradeFinsih();

    --Resume时不需要继续调用流程处理函数, 需要检查当前流程是否已经执行完成
    if (g_UpdState.FlowState == FLOW_STATE_RUNNING) then
        return FLOW_DISPATCH_STATE_NORMAL;
    elseif (g_UpdState.FlowState == FLOW_STATE_UPD_SUSPEND) then
        NotifyMmtWhenSuspend(true);
	    if (MODE_SHIELD_ALARM == g_UpdState.ParamList.shieldalarmtype) then
            NotifyAlarmShield(false);                --专家模式恢复流程的时候屏蔽告警
        end
        return FLOW_DISPATCH_STATE_UPD_SUSPEND;
    elseif (g_UpdState.FlowState == FLOW_STATE_RBK_SUSPEND) then
        NotifyMmtWhenSuspend(true);
	    if (MODE_SHIELD_ALARM == g_UpdState.ParamList.shieldalarmtype) then
           NotifyAlarmShield(false);                --专家模式恢复流程的时候屏蔽告警
        end
        return FLOW_DISPATCH_STATE_RBK_SUSPEND;
    elseif (g_UpdState.FlowState == FLOW_STATE_END_SUSPEND) then
        NotifyMmtWhenSuspend(true);
	    if (MODE_SHIELD_ALARM == g_UpdState.ParamList.shieldalarmtype) then
            NotifyAlarmShield(false);                --专家模式恢复流程的时候屏蔽告警
        end
        return FLOW_DISPATCH_STATE_END_SUSPEND;
    else
        return FLOW_STATE_FINISH;
    end
end

-------------------------------------------------------------------------------
-- Func Name       : GetFlowInfo
-- Description     : 查询指定流程包含的节点类型和AtomGroup信息
-- Caution         : 由升级框架调用，函数名称、参数、返回值必须与模板一致
--------------------------------------------------------------------------------
function GetFlowInfo(Flow)
    return g_UpdateFlowsTbl[TransFlowName(Flow)].info;
end

-------------------------------------------------------------------------------
-- Func Name       : GetResumeFlowName
-- Description     : 获取要恢复的流程名称
-- Caution         : 由升级框架调用，函数名称、参数、返回值必须与模板一致
--------------------------------------------------------------------------------
function GetResumeFlowName()
    if ((nil == g_UpdState.Flow) or ("" == g_UpdState.Flow)) then
        LogText(LL_NOTICE, "No flow is executing.");
        return "";
    end

    return g_UpdState.Flow;
end

-------------------------------------------------------------------------------
-- Func Name       : GetResumeNodeList
-- Description     : 恢复升级的节点列表
-- Caution         : 由升级框架调用，函数名称、参数、返回值必须与模板一致
--------------------------------------------------------------------------------
function GetResumeNodeList()
    if ((nil == g_UpdState) or (nil == g_UpdState.Flow)
        or ("" == g_UpdState.Flow) or (nil == g_UpdState.NodeList)) then
        LogText(LL_NOTICE, "No flow is executing.");
        return "";
    end

    local nodelist = "";
    for _, Node in pairs(g_UpdState.NodeList) do
        nodelist = nodelist .. Node.ip .. ",";
    end

    return nodelist;
end

-------------------------------------------------------------------------------
-- Func Name       : GetResumePackageType
-- Description     : 恢复导包流程的包类型
-- Caution         : 由升级框架调用，函数名称、参数、返回值必须与模板一致
--------------------------------------------------------------------------------
function GetResumePackageType()
    if ((nil == g_UpdState.Flow) or ("" == g_UpdState.Flow) or ("DownloadPkg" ~= g_UpdState.Flow)) then
        LogText(LL_NOTICE, "No flow is executing.");
        return "";
    end
    return g_UpdState.ParamList.pkgpath;
end

--------------------------------------------------------------------------------
-- Func Name       : GetFlowId
-- Description     : 获取当前正在执行的流程的序列号
-- Caution         : 由升级框架调用，函数名称、参数、返回值必须与模板一致
-- Input           :
-- Output          :
-- Return          : 流程的流水号
--------------------------------------------------------------------------------
function GetFlowId()
    return g_UpdState.FlowId;
end


--------------------------------------------------------------------------------
-- Func Name       : IsAtomGroupFinished
-- Description     : 判断节点当前执行的AG是否完成,根据AG的状态判断是否更新原子信息
-- Caution         : 由升级框架调用，函数名称、参数、返回值必须与模板一致
-- Input           : NodeIp：string 节点ip
--                 ：AtomGroupName：string 原子组名称
-- Return          : TRUE：执行完成
--                 ：FALSE：未执行完成
--------------------------------------------------------------------------------
function IsAtomGroupFinished(NodeIp, AtomGroupName)

    if (AtomGroupName == "AbnormalReboot")then
        return FALSE;
    end

    if (g_UpdState.FlowState == FLOW_STATE_FINISH) then
        return TRUE
    end
    --此处的判断主要针对升级的场景,不考虑同步的场景,所以不考虑MasterList
    local Node = g_UpdState.NodeList[ip2int(NodeIp)]
    if (Node == nil) then
        return TRUE
    end

    if ((AtomGroupName == Node.ag) and (Node.rt == ATOM_GROUP_EXEC_RET_INIT))then
        LogText(LL_NOTICE, "The atom group has not exec finished." );
        return FALSE
    end

    LogText(LL_NOTICE, "Node finished the AG or node has finished:(AtomGroupName)and(Node.ag)" .. AtomGroupName .."and" .. Node.ag .. "(Node.st)" ..Node.st);
    return TRUE
end

function IsAtomGroupConCurrent(AGName)
    local isAGConCurrent = {
                                ["UpgradeSystemWhenOnlineUpgrade"]=true,
                                ["UpgradePostupd"]=true,
                           };

    if(isAGConCurrent[AGName])then
        LogText(LL_NOTICE, "The AG is conCunrrent " .. AGName);
        return TRUE;
    end
    LogText(LL_NOTICE, "The AG is not conCunrrent " .. AGName);
    return FALSE;
end

--获取当前流程类型: 0--升级 1--回退 -1--错误
--------------------------------------------------------------------------------
-- Func Name       : GetFlowType
-- Description     : 获取当前升级的类型
-- Caution         : 由升级框架调用，函数名称、参数、返回值必须与模板一致
-- Input           :
-- Output          :
-- Return          : 0--升级 1--回退 -1--错误
--------------------------------------------------------------------------------
function GetFlowType(Flow)
    if (nil == Flow) or ("" == Flow) then
        return -1;
    end

    if g_UpdateFlowsTbl[Flow].isRbk then
        return 1;
    else
        return 0;
    end
end

--------------------------------------------------------------------------------
-- Func Name       : GetFlowTimeout
-- Description     : 获取指定流程的超时时间
-- Caution         : 由升级框架调用，函数名称、参数、返回值必须与模板一致
-- Input           : FlowName: string 流程名称
-- Output          :
-- Return          : 返回值1: int 执行结果, RET_OK: 成功, RET_ERR: 失败
--                   返回值2: int 流程的超时时间
--------------------------------------------------------------------------------
function GetFlowTimeout(FlowName)
    local flow = g_UpdateFlowsTbl[FlowName];
    local timeout = flow.timeout;
    LogText(LL_NOTICE, "GetFlowTimeout: flow=" .. FlowName .. " timeout=" .. timeout);
    if (flow == nil) or (timeout == nil) then
        return RET_ERR, timeout;
    else
        return RET_OK, timeout;
    end
end

--------------------------------------------------------------------------------
-- Func Name       : GetFlowProgress
-- Description     : 获取当前流程的进度信息
-- Caution         : 由升级框架调用，函数名称、参数、返回值必须与模板一致
-- Input           :
-- Output          :
-- Return          : 返回值1: int 执行结果, RET_OK: 成功, RET_ERR: 失败
--                   返回值2: string 流程的进度信息
--------------------------------------------------------------------------------
function GetFlowProgress()
    if (g_UpdState == nil) or (g_UpdState.NodeList == nil) then
        return RET_ERR, nil;
    else
        --将table: {[112]={ip=112,tp='OMS',st=1,rt=1,bt=1111,et=1111,pr=10},{}}
        --转换成字符串: {"112":{"ip":111,"tp":"OMS","st":1,"rt":1,"bt":1111,"et":1111,"pr":10},{}}
        local s = "{";
        for key, Node in pairs(g_UpdState.NodeList) do
            s = s .. "\"" .. key .. "\":{\"ip\":\"" .. Node.ip .. "\",\"tp\":\"" ..
                Node.tp .. "\",\"st\":" .. Node.st .. ",\"rt\":" .. Node.rt ..
                ",\"bt\":" .. Node.bt .. ",\"et\":" .. Node.et .. ",\"pr\":" ..
                Node.pr .. "},";
        end
        s = string.sub(s, 1, -2) .. "}"; --去掉最后一个元素后面的","

        LogText(LL_NOTICE, "GetFlowProgress: " .. s);
        return RET_OK, s;
    end
end

--------------------------------------------------------------------------------
-- Func Name       : GetFlowTotalTime
-- Description     : 获取流程的总时间
-- Caution         : 由升级框架调用，函数名称、参数、返回值必须与模板一致
-- Input           :
-- Output          :
-- Return          : 返回值1: int 执行结果, RET_OK: 成功, RET_ERR: 失败
--                   返回值2: string 流程的进度信息
--------------------------------------------------------------------------------
function GetFlowTotalTime()
    if ((nil == g_UpdState.Flow) or ("" == g_UpdState.Flow)) then
        LogText(LL_NOTICE, "No flow is executing.");
        return RET_ERR;
    end

    local totalTime = g_UpdateFlowsTbl[g_UpdState.Flow].totalTime;

    if (nil == totalTime) then
        LogText(LL_NOTICE, "The para is not config.");
        return RET_ERR;
    end

    return RET_OK, g_UpdateFlowsTbl[g_UpdState.Flow].totalTime;
end

--------------------------------------------------------------------------------
-- Func Name       : Table2String
-- Description     : 将table转换成string
-- Caution         : 由OMM维护, 请勿修改
-- Input           :
-- Output          :
-- Return          :
--------------------------------------------------------------------------------
function Table2String(paramTable)
    local param = "{"
    for key, value in pairs(paramTable) do
        param = param .. key .. "='" .. value .. "',"
    end
    param = param .. "}"
    return param
end

--------------------------------------------------------------------------------
-- Func Name       : UpdateNodeState
-- Description     : 更新节点的升级进度
-- Caution         : 由升级框架调用，函数名称、参数、返回值必须与模板一致
-- Input           : ActionType: int 节点状态更新事件类型
--                   NodeList: table 待更新的节点列表
--                   NodeType: string 节点类型
-- Output          :
-- Return          :
--------------------------------------------------------------------------------
function UpdateNodeState(EventType, NodeList, NodeType)
    LogText(LL_NOTICE, "UpdateNodeState, EventType: " .. EventType);

    local IsRbk = g_UpdateFlowsTbl[g_UpdState.Flow].isRbk;

    if (EventType == START_BATCH) then
        for _, Node in pairs(NodeList) do
            g_UpdState.NodeList[ip2int(Node.ip)].st = NODE_STATE_RUNNING;
            g_UpdState.NodeList[ip2int(Node.ip)].bt = os.time();
        end
    elseif (EventType == END_BATCH) then
        for _, Node in pairs(NodeList) do
            if IsRbk then --回退流程
                if (Node.state == ATOM_GROUP_EXEC_RET_SUCC_RBK) then
                    g_UpdState.NodeList[ip2int(Node.ip)].st = NODE_STATE_SUCC;
                elseif (Node.state == ATOM_GROUP_EXEC_RET_FAIL) then
                    g_UpdState.NodeList[ip2int(Node.ip)].st = NODE_STATE_FAIL;
                elseif (Node.state == ATOM_GROUP_EXEC_RET_TIMEOUT) then
                    g_UpdState.NodeList[ip2int(Node.ip)].st = NODE_STATE_TIMEOUT;
                end
            else --升级流程
                if (Node.state == ATOM_GROUP_EXEC_RET_SUCC) then
                    g_UpdState.NodeList[ip2int(Node.ip)].st = NODE_STATE_SUCC;
                elseif (Node.state == ATOM_GROUP_EXEC_RET_SUCC_RBK) then
                    g_UpdState.NodeList[ip2int(Node.ip)].st = NODE_STATE_SUCC_RBK;
                elseif (Node.state == ATOM_GROUP_EXEC_RET_FAIL) then
                    g_UpdState.NodeList[ip2int(Node.ip)].st = NODE_STATE_FAIL;
                elseif (Node.state == ATOM_GROUP_EXEC_RET_TIMEOUT) then
                    g_UpdState.NodeList[ip2int(Node.ip)].st = NODE_STATE_TIMEOUT;
                elseif (Node.state == ATOM_GROUP_EXEC_RET_PAUSE) then
                    g_UpdState.NodeList[ip2int(Node.ip)].st = NODE_STATE_CANCEL;
                end
            end

            g_UpdState.NodeList[ip2int(Node.ip)].et = os.time();
        end
    elseif (EventType == END_FLOW) then
        --流程结束时, 未开始升级的节点状态修改为被自动取消, 不需要刷新开始、结束时间
        for _, Node in pairs(g_UpdState.NodeList) do
            if (Node.st == NODE_STATE_INIT) then
                Node.st = NODE_STATE_CANCEL;
            end
        end
    elseif (EventType == TERM_FLOW) then
        --流程超时时, 未完成的节点状态修改为超时
        for _, Node in pairs(g_UpdState.NodeList) do
            if (Node.st == NODE_STATE_INIT) then
                Node.st = NODE_STATE_CANCEL;
            elseif (Node.st == NODE_STATE_RUNNING) then
                Node.st = NODE_STATE_TIMEOUT;
                Node.et = os.time();
            end
        end
    end
end

--------------------------------------------------------------------------------
-- Func Name       : GetSizeOfTable
-- Description     : 获取记录表的元素个数
-- Caution         :
-- Input           : NodeTable: table 要获取的表名

-- Output          : 元素的个数
-- Return          : 元素的个数
--------------------------------------------------------------------------------
function GetSizeOfTable(NodeTable)
    local num = 0;
    for _, Node in pairs(NodeTable) do
        num = num + 1;
    end
    return num;
end

--------------------------------------------------------------------------------
-- Func Name       : GetNodeIndex
-- Description     : 根据Nid获取节点在主备信息表中的下标
-- Caution         : 当前只是在多控获取批次信息时使用，热补丁异常场景可能会返回nil
-- Input           : FirstNid: string 节点的nid
-- Output          : 节点下标
-- Return          : 节点下标
--------------------------------------------------------------------------------
function GetNodeIndex(FirstNid)
    LogText(LL_NOTICE, "GetNodeIndex, FirstNid: " .. FirstNid);
    for key, Node in pairs(g_UpdState.g_MasterSlave) do
        if(Node.Nid == FirstNid) then
            LogText(LL_NOTICE, "GetNodeIndex, Index: " .. key);
            return key;
        end
    end
    LogText(LL_NOTICE, "GetNodeIndex, FirstNid:" .. FirstNid .. " return nil.");
    return nil;
end

--------------------------------------------------------------------------------
-- Func Name       : IsNotInFirstBatch
-- Description     : 根据ip判断节点是否不在第一批次
-- Caution         : 由于当前双控多控获取批次信息时经常用到判断节点是否在第一批次所以此处封装
-- Input           : ip: string 节点的ip
-- Output          : false：节点在第一批次 true：节点不在第一批次
-- Return          : false：节点在第一批次 true：节点不在第一批次
--------------------------------------------------------------------------------
function IsNotInFirstBatch(ip)
    LogText(LL_NOTICE, "IsNotInFirstBatch: " .. ip);
    for key, Node in pairs(g_UpdState.NodeList) do
        if(Node.ip == ip and Node.batchno == 1) then
            LogText(LL_NOTICE, "Node in batchno1.");
            return false;
        end
    end
    LogText(LL_NOTICE, "Node not in batchno1.");
    return true;
end

--------------------------------------------------------------------------------
-- Func Name       : IsInLastBatch
-- Description     : 根据节点的ip判断节点是否在最后一个批次
-- Caution         : 根据节点的ip判断节点是否在最后一个批次
-- Input           : ip: string 节点的ip
-- Output          : true: 节点在最后一个批次 false: 节点不在最后一个批次
-- Return          : true: 节点在最后一个批次 false: 节点不在最后一个批次
--------------------------------------------------------------------------------
function IsInLastBatch(ip)
    LogText(LL_NOTICE, "IsInLastBatch: " .. ip);
    for key, Node in pairs(g_UpdState.NodeList) do
        if(Node.ip == ip and (Node.batchno ==1 or Node.batchno == 2)) then
            LogText(LL_NOTICE, "Node in batchno1 or batchno2.");
            return false;
        end
    end
    LogText(LL_NOTICE, "Node not in batchno1 or batchno2.");
    return true;
end

--------------------------------------------------------------------------------
-- Func Name       : GetNodeListByPlugin
-- Description     : 根据输入的批次获取当前批次升级的节点信息
-- Caution         : 根据输入的批次获取当前批次升级的节点信息
-- Input           : BatchNo: 升级的批次
--                   RbkBatchNo：回退的批次
-- Output          : 升级批次的节点信息
--                 ：回退批次的节点信息
-- Return          : 升级批次的节点信息
--                 ：回退批次的节点信息
--------------------------------------------------------------------------------
function GetNodeListByPlugin(BatchNo,RbkBatchNo)
    if(0 == g_UpdState.FlowUpdState)then
        return GetUpdNodeListByPlugin(BatchNo);
    else
        return GetRbkNodeListByPlugin(RbkBatchNo);
    end
end

--------------------------------------------------------------------------------
-- Func Name       : GetNodeListNotInBatches
-- Description     : 根据批次获取执行升级的节点信息
-- Caution         : 此函数适用于不需要分批次的flow 第一批次为所有节点 第二批次为nil
-- Input           : BatchNo: 升级的批次
-- Output          : 升级批次的节点信息
-- Return          : 升级批次的节点信息
--------------------------------------------------------------------------------
function GetNodeListNotInBatches(BatchNo)
    LogText(LL_NOTICE, "GetNodeListNotInBatches,BatchNo: " .. BatchNo);
    local FirstBatch = {};
    local SecondBatch = {};
    if (1 == BatchNo)then
        for _, Node in pairs(g_UpdState.NodeList) do
            table.insert(FirstBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 1});
        end
        return FirstBatch;
    else
        LogText(LL_NOTICE, "Batch Number is false, and SecondBatch=Null." );
        SecondBatch = nil;
        return SecondBatch;
    end
end

--------------------------------------------------------------------------------
-- Func Name       : GetIpByNid
-- Description     : 根据nid获取节点ip
-- Caution         : g_UpdState.g_ClusterInfo中的Nid是字符串
-- Input           :
-- Output          : 节点的ip
--------------------------------------------------------------------------------
function GetIpByNid(nid)
    LogText(LL_NOTICE, "GetIpByNid" .. nid );
    for Index, Node in pairs(g_UpdState.g_ClusterInfo) do
        if (Node.Nid == nid) then
            LogText(LL_NOTICE, "GetIpByNid" .. Node.ip);
            return  Node.ip;
        end
    end
    LogText(LL_NOTICE, "GetIpByNid has a wrong nid" .. nid );
    return nil;
end
--------------------------------------------------------------------------------
-- Func Name       : InitClusterTypeByCfg
-- Description     : 根据配置文件初始化集群的类型
-- Caution         : 配置文件中标记批次的标签是:Batch,标记升级节点的标签是:NodeList
-- Input           :
-- Output          : 节点的nid
--------------------------------------------------------------------------------
function InitClusterTypeByCfg()
    local head="<".."Batch"..">"
    local tail="</".."Batch"..">"
    local fn = "/OSM/script/UpgradeFile.xml"
    local f = io.open(fn, "r");
    if (nil == f) then
        LogText(LL_WARN, "No need read upgrade xml file.");
        return;
    end
    local data = f:read("*all");
    f:close();
    _,_,value=string.find(data, head .. "(.-)" .. tail);
    if('0' == value)then
        --一个一个节点升级
        g_UpdState.g_ClusterType = BATCH_BY_CLUSTRINFO;
        g_UpdState.g_MaxBatchNo = #(g_UpdState.g_ClusterInfo);
        g_UpdState.g_CfgBatchNode={};
    else
        --按照配置参数执行升级
        g_UpdState.g_ClusterType = BATCH_BY_CONFIG;
        --从lua中读取的变量是字符型，通过+0将其转换成数字
        g_UpdState.g_MaxBatchNo = value + 0;
        --获取各个批次配置的节点信息
        head="<".."NodeList"..">";
        tail="</".."NodeList"..">";
        _,_,value=string.find(data, head .. "(.-)" .. tail);
        --最后一个批次没有加分隔符;首先加一个方便后续处理
        g_UpdState.g_CfgBatchNode = value .. ";";
        local index = string.find(g_UpdState.g_CfgBatchNode, "C");
        if(index ~= nil)then
            g_UpdState.g_GetNidType = V3_6U_NID_TYPE;
        end
        LogText(LL_NOTICE, "Get config nodelist is and nid type:" .. g_UpdState.g_CfgBatchNode .. g_UpdState.g_GetNidType);
    end
    LogText(LL_NOTICE, "Get config batchno and cluster info:" .. g_UpdState.g_ClusterType .. "g_MaxBatchNo:" .. g_UpdState.g_MaxBatchNo);
end
--------------------------------------------------------------------------------
-- Func Name       : GetNodeNidByCfg
-- Description     : 根据配置的节点信息获取节点的nid
-- Caution         : 节点的nid及配置的节点信息都是字符串
-- Input           : cfgnid: 配置的节点信息
-- Output          : 节点的nid
--------------------------------------------------------------------------------
function GetNodeNidByCfg(cfgnid)
    local CfgToNid;
    if(V3_6U_NID_TYPE == g_UpdState.g_GetNidType)then
        CfgToNid = {["0A"]="0",  ["0B"]="1",  ["0C"]="2",  ["0D"]="3",
                    ["1A"]="4",  ["1B"]="5",  ["1C"]="6",  ["1D"]="7",
                    ["2A"]="8",  ["2B"]="9",  ["2C"]="10", ["2D"]="11",
                    ["3A"]="12", ["3B"]="13", ["3C"]="14", ["3D"]="15"};
    else
        CfgToNid = {["0A"]="0",  ["0B"]="1",  ["1A"]="2",  ["1B"]="3",
                    ["2A"]="4",  ["2B"]="5",  ["3A"]="6",  ["3B"]="7",
                    ["4A"]="8",  ["4B"]="9",  ["5A"]="10", ["5B"]="11",
                    ["6A"]="12", ["6B"]="13", ["7A"]="14", ["7B"]="15"};
    end
    for k, v in pairs(CfgToNid) do
        if(cfgnid == k) then
            LogText(LL_NOTICE, "GetNodeNidByCfg:" .. v);
            return v;
        end
    end
    LogText(LL_NOTICE, "GetNodeNidByCfg return nil." );
    return nil
end
--------------------------------------------------------------------------------
-- Func Name       : GetNodeListByCfgXml
-- Description     : 根据XML配置文件中的节点信息获取各个批次节点
-- Caution         : 批次信息格式如下:<NodeList>1A;0B,1B;2B,3B;0A,2A,3A</ NodeList>
--                   其中批次之间用分号分隔,节点之间用逗号分隔每个批次最后一个节点添加一个逗号分隔处理
-- Input           : BatchNo: 升级的批次
-- Output          : 升级批次的节点信息
-- Return          : 升级批次的节点信息
--------------------------------------------------------------------------------
function GetNodeListByCfgXml(BatchNo)
    if(BatchNo > g_UpdState.g_MaxBatchNo)then
        LogText(LL_NOTICE, "The batch number is invaild:" .. BatchNo);
        return nil;
    end
    if(1 == BatchNo)then
        ---第一批次执行时，首先初始化一下节点的批次信息
        for _, Node in pairs(g_UpdState.NodeList) do
            g_UpdState.NodeList[ip2int(Node.ip)].batchno = 0;
        end
    end
    local BatchList = {};
    local index = string.find(g_UpdState.g_CfgBatchNode, ";")
    if(nil ~= index)then
        --依次获取当前批次节点信息,首先在最后一个节点后面加一个分隔符","方便处理
        local list = string.sub(g_UpdState.g_CfgBatchNode, 1, index - 1);
        list = list .. ","
        for w in string.gmatch(list, "%d.-,") do
            node = string.gsub(w, "," , "");
            nid = GetNodeNidByCfg(node);
            Nodeip = GetIpByNid(nid);
            table.insert(BatchList,{ip = Nodeip, type = "Manager", state = ATOM_GROUP_EXEC_RET_INIT, batchno = BatchNo});
            g_UpdState.NodeList[ip2int(Nodeip)].batchno = BatchNo;
            LogText(LL_NOTICE, "GetNodeListByCfgXml nid:" .. nid);
        end
        --获取后更新要升级的节点列表删除当前批次节点
        g_UpdState.g_CfgBatchNode = string.sub(g_UpdState.g_CfgBatchNode, index + 1);
    end
    SaveState(g_UpdState);
    return BatchList;
end
--------------------------------------------------------------------------------
-- Func Name       : GetOneNodeByBatchNo
-- Description     : 一个节点一个节点升级保证最后升级主控
-- Caution         : 按照nid从小到大依次执行升级
-- Input           : BatchNo: 升级的批次
-- Output          : 升级批次的节点信息
--------------------------------------------------------------------------------
function GetOneNodeByBatchNo(BatchNo)
    if(BatchNo > g_UpdState.g_MaxBatchNo)then
        LogText(LL_NOTICE, "The batch number is invaild:" .. BatchNo);
        return nil;
    end
    local batchno;
    local BatchList = {};
    if(1 == BatchNo)then
        ---第一批次执行时，首先初始化一下节点的批次信息
        for _, Node in pairs(g_UpdState.NodeList) do
            g_UpdState.NodeList[ip2int(Node.ip)].batchno = 0;
        end
    end
    if(g_UpdState.g_MaxBatchNo == BatchNo)then
        --最后一个批次时，升级主控
        for Index, Node in pairs(g_UpdState.g_ClusterInfo) do
            batchno = g_UpdState.NodeList[ip2int(Node.ip)].batchno;
            if(0 == batchno)then
                table.insert(BatchList,{ip = Node.ip, type = "Manager", state = ATOM_GROUP_EXEC_RET_INIT, batchno = BatchNo});
                g_UpdState.NodeList[ip2int(Node.ip)].batchno = BatchNo;
                LogText(LL_NOTICE, "GetOneNodeByBatchNo nid:" .. Node.Nid);
                SaveState(g_UpdState);
                return BatchList;
            end
        end
    end
    --最多支持16控，最大的nid是15
    local minNid = 16;
    local minnode;
    for Index, Node in pairs(g_UpdState.g_ClusterInfo) do
        batchno = g_UpdState.NodeList[ip2int(Node.ip)].batchno;
        --选择非主节点中nid最小的节点最为当前批次要升级的节点信息 nid是字符串加0转化为数字
        if (((Node.Nid+0) < minNid) and (0 == batchno) and (Node.ip ~= GetMasterIp())) then
            minNid = (Node.Nid+0);
            minnode = Node;
        end
    end
    table.insert(BatchList,{ip = minnode.ip, type = "Manager", state = ATOM_GROUP_EXEC_RET_INIT, batchno = BatchNo});
    g_UpdState.NodeList[ip2int(minnode.ip)].batchno = BatchNo;
    LogText(LL_NOTICE, "GetOneNodeByBatchNo nid:" .. minnode.Nid);
    SaveState(g_UpdState);
    return BatchList;
  end
--------------------------------------------------------------------------------
-- Func Name       : GetOneNodeByBatchNo
-- Description     : 一个节点一个节点升级保证最后升级主控
-- Caution         : 按照nid从小到大依次执行升级
-- Input           : BatchNo: 升级的批次
-- Output          : 升级批次的节点信息
--------------------------------------------------------------------------------
function GetNodeListByDefalutBatch(BatchNo)
    if(BatchNo > g_UpdState.g_MaxBatchNo)then
        LogText(LL_NOTICE, "The batch number is invaild:" .. BatchNo);
        return nil;
    end
    local batchno;
    local BatchList = {};
    if(1 == BatchNo)then
        ---第一批次执行时，首先初始化一下节点的批次信息
        for _, Node in pairs(g_UpdState.NodeList) do
            g_UpdState.NodeList[ip2int(Node.ip)].batchno = 0;
        end
        ---第一批次升级与主控不在一个平面的节点
        for _, Node in pairs(g_UpdState.g_ClusterInfo) do
            if ((Node.Nid%2) ~= (GetMasterNid()%2)) then
                LogText(LL_NOTICE, "Insert FirstBatch node:" .. Node.Nid);
                table.insert(BatchList,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 1});
                g_UpdState.NodeList[ip2int(Node.ip)].batchno = 1;
            end
        end
    elseif(2 == BatchNo)then
        for Index, Node in pairs(g_UpdState.g_ClusterInfo) do
            batchno = g_UpdState.NodeList[ip2int(Node.ip)].batchno;
            if(0 == batchno)then
                LogText(LL_NOTICE, "Insert SecondBatch node:" .. Node.Nid);
                table.insert(BatchList,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 2});
                g_UpdState.NodeList[ip2int(Node.ip)].batchno = 2;
            end
        end
    end
    SaveState(g_UpdState);
    return BatchList;
end
--------------------------------------------------------------------------------
-- Func Name       : GetUpdNodeListByPlugin
-- Description     : 根据批次获取执行升级的节点信息
-- Caution         : 此函数为获取升级批次信息的总函数既包括不需要分批的也包括需要分批次的flow
-- Input           : BatchNo: 升级的批次
-- Output          : 升级批次的节点信息
-- Return          : 升级批次的节点信息
--------------------------------------------------------------------------------
function GetUpdNodeListByPlugin(BatchNo)
    LogText(LL_NOTICE, "GetNodeListByPlugin, BatchNo: " .. BatchNo);

    local FirstBatch = {};
    local SecondBatch = {};
    local ThirdBatch = {};
    local MasterNid = "0";
    local Index = 0;

    --不需要分批次执行的flow
    if(not g_UpdateFlowsTbl[g_UpdState.Flow].isInBatches) then
        return GetNodeListNotInBatches(BatchNo);
    end

    if(BATCH_BY_CONFIG == g_UpdState.g_ClusterType) then
        return GetNodeListByCfgXml(BatchNo);
    end
    if(BATCH_BY_CLUSTRINFO == g_UpdState.g_ClusterType) then
        return GetOneNodeByBatchNo(BatchNo);
    end

    if(BATCH_BY_DEFAULT == g_UpdState.g_ClusterType)then
        return GetNodeListByDefalutBatch(BatchNo);
    end
    -- 如果是单控，只有第一批有节点
    if(GetSizeOfTable(g_UpdState.NodeList) == 1 and #g_UpdState.g_MasterSlave == 1) then
        LogText(LL_NOTICE, "This is single controller.");
        if(BatchNo == 1)then
            g_UpdState.g_ClusterType = 1;
            g_UpdState.g_MaxBatchNo = 1;
             --单控第一批为所有的节点，其余为NIL
            for _, Node in pairs(g_UpdState.NodeList) do
                table.insert(FirstBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 1});
                g_UpdState.NodeList[ip2int(Node.ip)].batchno = 1;
            end
            return FirstBatch;
        else
            return nil;
        end
    --需要分批次执行flow时，根据双控和多控采用不同的分批策略
    elseif((GetSizeOfTable(g_UpdState.NodeList) == 2 and #g_UpdState.g_MasterSlave == 2) or (g_UpdState.g_ClusterType == 2)) then  ---在线升级不支持
        LogText(LL_NOTICE, "This is double controller.");
        if(BatchNo == 1)then
            g_UpdState.g_ClusterType = 2;
            g_UpdState.g_MaxBatchNo = 2;
            --初始化节点的批次信息
            for _, Node in pairs(g_UpdState.NodeList) do
                g_UpdState.NodeList[ip2int(Node.ip)].batchno = 0;
            end
            --双控先升级备再升级主，第一批次获取备控节点
            for _, Node in pairs(g_UpdState.NodeList) do
                if (not IsMasterNode(Node.ip)) then
                    table.insert(FirstBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 1});
                    g_UpdState.NodeList[ip2int(Node.ip)].batchno = 1;
                end
            end
            return FirstBatch;
        --不在第一批次的节点为第二批次节点，防止升级过程中发生主备倒换
        elseif(2 == BatchNo)then
            for _, Node in pairs(g_UpdState.g_ClusterInfo) do
                if (IsNotInFirstBatch(Node.ip)) then
                    LogText(LL_NOTICE, "Insert SecondBatch node:" .. Node.Nid);
                    table.insert(SecondBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 2});
                    g_UpdState.NodeList[ip2int(Node.ip)].batchno = 2;
                end
            end
            return SecondBatch;
        else
            LogText(LL_NOTICE, "GetUpdNodeListByPlugin:Batch Number is invalid." .. BatchNo);
            return nil;
        end
    else
        LogText(LL_NOTICE, "This is multi controller.");
        g_UpdState.g_ClusterType = 3;
        g_UpdState.g_MaxBatchNo = 3;
        if (BatchNo == 1)then
            ---第一批次执行时，首先初始化一下节点的批次信息,热补丁执行第二个原子组时,会再次调用这个函数.
            for _, Node in pairs(g_UpdState.NodeList) do
                g_UpdState.NodeList[ip2int(Node.ip)].batchno = 0;
            end
            for _, Node in pairs(g_UpdState.g_MasterSlave) do
                if (Node.IsMaster == "1" ) then
                    if(Node.Nid == "0" or Node.Nid == "1") then
                        FirstNid = "2";
                    else
                        FirstNid = "0";
                    end
                    Index = GetNodeIndex(FirstNid);
                    if(nil == Index)then
                        LogText(LL_NOTICE, "The FirstBatch node may reboot.");
                        return nil;
                    end
                    CurNode =  g_UpdState.g_MasterSlave[Index];
                    LogText(LL_NOTICE, "Insert FirstBatch node:" .. CurNode.Nid);
                    table.insert(FirstBatch,{ip = CurNode.ip, type = CurNode.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 1});
                    g_UpdState.NodeList[ip2int(CurNode.ip)].batchno = 1;
                end
            end
        end

        for _, Node in pairs(g_UpdState.g_ClusterInfo) do
            if ((BatchNo == 2) and ((Node.Nid%2) ~= (GetMasterNid()%2)) and IsNotInFirstBatch(Node.ip)) then ---
                LogText(LL_NOTICE, "Insert SecondBatch node:" .. Node.Nid);
                table.insert(SecondBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 2});
                g_UpdState.NodeList[ip2int(Node.ip)].batchno = 2;
            elseif((BatchNo == 3) and IsInLastBatch(Node.ip)) then--进行到第三批次说明前两个批次节点都成功
                LogText(LL_NOTICE, "Insert ThirdBatch node:" .. Node.Nid);
                table.insert(ThirdBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 3});
                g_UpdState.NodeList[ip2int(Node.ip)].batchno = 3;
            end
        end
    end
    SaveState(g_UpdState);

    if(BatchNo == 1) then
        return FirstBatch;
    elseif(BatchNo == 2) then
        return SecondBatch;
    elseif(BatchNo == 3) then
        return ThirdBatch;
    else
        LogText(LL_NOTICE, "GetNodeListByPlugin:Batch Number is invalid." );
        return nil;
    end
end

--------------------------------------------------------------------------------
-- Func Name       : GetRbkNodeListByPlugin
-- Description     : 根据回退批次获取回退的节点
-- Caution         : 根据回退批次获取回退的节点
-- Input           : RbkBatchNo: 回退的批次
-- Output          : 回退批次的节点信息
-- Return          : 回退批次的节点信息
--------------------------------------------------------------------------------
function GetRbkNodeListByPlugin(RbkBatchNo)
    LogText(LL_NOTICE, "GetRbkNodeListByPlugin, RbkBatchNo: " .. RbkBatchNo);
    local RbkCurBatch = {};
    --获取当前要回退的批次对应的之前升级过的批次
    local RbkUpdBatch = g_UpdState.BatchNo + 1 - RbkBatchNo;

    --不需要分批次回退的flow，直接所有的升级节点，已经回退的空转，否则执行回退原子
    if(not g_UpdateFlowsTbl[g_UpdState.Flow].isInBatches) then
        return GetNodeListNotInBatches(RbkBatchNo);
    end

    --需要分批次执行的flow，首先回退当前批次升级的节点，然后再依次回退其他批次
    if(0 >= RbkUpdBatch)then
        LogText(LL_NOTICE, "GetRbkNodeListByPlugin:Batch Number is invalid." .. RbkBatchNo);
        return nil;
    end

    --根据节点升级的批次依次回退
    for _, Node in pairs(g_UpdState.NodeList) do
        if(RbkUpdBatch == g_UpdState.NodeList[ip2int(Node.ip)].batchno)then
            table.insert(RbkCurBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT});
        end
    end
    return RbkCurBatch;
end

--------------------------------------------------------------------------------
-- Func Name       : IsFinishAllAG
-- Description     : 是否完执行完成所有的原子组
-- Caution         : 此处要注意区分当前的升级状态是升级还是升级失败后的回退
-- Input           : AGIndex: 当前执行的AGIndex
-- Output          : true：完成所有AGIndex false：未完成所有AGIndex
-- Return          : true：完成所有AGIndex false：未完成所有AGIndex
--------------------------------------------------------------------------------
function IsFinishAllAG(AGIndex)
    AGIndex = UpdateTheLastAGIndex(AGIndex);
    if(0 == g_UpdState.FlowUpdState and AGIndex >= #g_UpdateFlowsTbl[g_UpdState.Flow].upd) or
      (1 == g_UpdState.FlowUpdState and AGIndex >= #g_UpdateFlowsTbl[g_UpdState.Flow].rbk) then
        LogText(LL_NOTICE, "IsFinishAllAG return true.");
        return true;
    end
    return false;
end

--------------------------------------------------------------------------------
-- Func Name       : IsTheFailRatioNeedRbk
-- Description     : 根据当前的失败率判断是否需要执行大回退
-- Caution         : 当前适用于离线升级和在线升级
-- Input           : 无
-- Return          : true：需要执行大回退 false：不需要执行大回退
--------------------------------------------------------------------------------
function IsTheFailRatioNeedRbk()
    --离线升级流程, 根据升级失败率计算, 当升级失败率大于给定值时, 需要回退(不需要分批次执行的flow按照此方式)
    if("OfflineUpgrade" == g_UpdState.Flow)then
        --此时升级状态为升级中，可以直接获取upd中的原子组信息
        local AtomGroup;
        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.AtomGroupIndex];
        --目前流程中Manager执行的原子组失败后直接走回退流程
        if("Manager" == AtomGroup.ExcuteNodeType)then
            LogText(LL_NOTICE, "Manager upgrade fail and rbk succeed.");
            return true;
        end
        local FailRatio = (#g_UpdState.FailNodeList)/(GetSizeOfTable(g_UpdState.NodeList)) * 100;
        if(UPGRADE_FAIL_RATE <= FailRatio) then
            LogText(LL_NOTICE, "Some node offline upgrade faild and need rollback system.");
            return true;
        end
    end
    --线升级流程，根据当前是否存在失败节点(需要分批次执行的flow由于一个批次完成后就会判断回退,所以可以直接根据失败节点判断)
    if("OnlineUpgrade" == g_UpdState.Flow)then
        --在线升级多控非最后一个批次失败需要回退，最后一个批次不回退，双控第一批需要回退,第二批不需要回退
        if(0 == g_UpdState.g_FinishAllBatch)then
            LogText(LL_NOTICE, "Some node online upgrade faild and need rollback system.");
            return true;
        end
    end
    LogText(LL_NOTICE, "Some node upgrade faild but no need rollback system.");
    return false;
end


--------------------------------------------------------------------------------
-- Func Name       : SetMemRecoverFlagInvalid
-- Description     : 设置复位保内存标记无效
-- Caution         : 主要针对离线升级部分成功部分失败和在线升级第三批次失败场景
-- Input           : 无
-- Return          : true：设置成功 false：设置失败
--------------------------------------------------------------------------------
function SetMemRecoverFlagInvalid(NodeIp)
    LogText(LL_NOTICE, "Exec SetMemRecoverFlagInvalid.");
    iRet = ServerSendMsgToAgentExecCmd{node_ip = NodeIp, exec_cmd = "/OSM/script/upgrade_preck.sh -l"};

    if(iRet == 0)then
        LogText(LL_NOTICE, "SetMemRecoverFlagInvalid success.");
        return true;
    end
    return false;
end

--------------------------------------------------------------------------------
-- Func Name       : RebootClusterWhenOfflineUpgrade
-- Description     : 部分场景升级失败后重启集群场景
--------------------------------------------------------------------------------
function RebootClusterWhenOfflineUpgrade()
    LogText(LL_NOTICE, "---Exec RebootClusterWhenOfflineUpgrade---");
    local NodeNid = GetMasterNid();
    local cmd = "/OSM/bin/upgrade_atom -f OFFLINE_REBOOT -c '" .. NodeNid .."'";
    os.execute("sleep " .. 10);
    return System(cmd, 7);
end

--------------------------------------------------------------------------------
-- Func Name       : GetRebootNodes
-- Description     : 获取需要重启的失败节点
--------------------------------------------------------------------------------
function GetRebootNodes()
    LogText(LL_NOTICE, "Enter getRebootNodes." );
    local RebootNodeInfo = {};
    local RebootNodeList = "";
    local Nid;

    for _, Node in pairs(g_UpdState.FailNodeList) do
        if(SetMemRecoverFlagInvalid(Node.ip)) then
            Nid = GetNidByIp(Node.ip);
            table.insert(RebootNodeInfo,{ip = Node.ip, nid = Nid});
            RebootNodeList = RebootNodeList .. Nid .. "," ;
        end
    end

    if (string.len(RebootNodeList) ~= 0) then
        g_UpdState.g_RebootNodeList = RebootNodeList;
        LogText(LL_NOTICE, "Reboot fail node nid: " .. RebootNodeList);
        SaveState(g_UpdState);
        --通知升级SERVER显示操作结果
        ReportRebootFailNodeToServer{node_list = g_UpdState.g_RebootNodeList};
        return RebootNodeInfo;
    end

    return nil;
end

--------------------------------------------------------------------------------
-- Func Name       :RebootFailNode
-- Description     :重启失败节点
-- Caution         :
-- Input           :无
-- Output          :无
-- Return          :无
--------------------------------------------------------------------------------
function RebootFailNode(RebootNodes)
    LogText(LL_NOTICE, "Enter RebootFailNode." );
    local MasterNid = GetMasterNid();

    if (RebootNodes == nil) then
        return;
    end

    --重启非主节点
    for _, Node in pairs(RebootNodes) do
        if(Node.nid ~= MasterNid)then
            RebootNode(Node.ip, Node.nid);
        end
    end

    --重启集群主
    for _, Node in pairs(RebootNodes) do
        if(Node.nid == MasterNid)then
            RebootNode(Node.ip, Node.nid);
        end
    end
end

--------------------------------------------------------------------------------
-- Func Name       :RebootNode
-- Description     :重启节点
-- Caution         :
-- Input           :ip,Nid
-- Output          :无
-- Return          :无
--------------------------------------------------------------------------------
function RebootNode(ip, Nid)
    LogText(LL_NOTICE, "Enter RebootNode ip and Nid: " .. ip .. "and" .. Nid);

    ServerSendMsgToAgentExecCmd{node_ip = ip, exec_cmd = "/OSM/bin/upgrade_atom -f ONLINE_REBOOT -c "..Nid};
end

--调用系统命令 fsync 使内存中的数据刷到硬盘中,避免重启丢失
--使用System命令，防止fsync命令挂住
function SyncMemeryToDisk()
    LogText(LL_NOTICE, "Sync memery data to disk.");
    local ret;
    local cmd = "fsync /startup_disk/conf/conf_local/upd_state_flows.lua";
    ret=System(cmd, 2);
    LogText(LL_NOTICE, "End to sync memery data to disk, result is " .. ret);
    os.execute("sleep " .. 30);
end

--------------------------------------------------------------------------------
-- Func Name       :CleanBackDirAfterUpgradeFail
-- Description     :流程执行失败且不需要重启失败节点则需要删除BAK目录中的目标版本
-- Caution         :
-- Input           :无
-- Output          :无
-- Return          :无
--------------------------------------------------------------------------------
function CleanBackDirAfterUpgradeFail()
    --只有在线或离线升级需要清除BAK目录
    if(("OnlineUpgrade" ~= g_UpdState.Flow) and ("OfflineUpgrade" ~= g_UpdState.Flow))then
        return;
    end

    --只有升级失败才需要清除BAK目录：当前还是升级流程且没有失败节点才为成功，其他为失败
    if((0 == #g_UpdState.FailNodeList) and (0 == g_UpdState.FlowUpdState)) then
        return;
    end

    --不需要回退的情况下无需清除BAK目录
    if ((g_UpdState.g_FlowStage ~= nil) and (g_UpdState.g_FlowStage.stage == STAGE_SECOND_FLOW)) then
        if(false == IsTheFailRatioNeedRbk())then
            return;
        end
    end

    --开始清除BAK目录
    LogText(LL_NOTICE, "Flow finish and need to delete back dir.");

    local cmd = "/OSM/script/upgrade_pcie.sh -e";

    --清除集群主上BAK目录
    System(cmd, 3);

    --清除非主节点的BAK目录
    for Index, Node in pairs(g_UpdState.NodeList) do
        if (not IsMasterNode(Node.ip)) then
            LogText(LL_NOTICE, "--Exec DeleteBackDir---");
            ServerSendMsgToAgentExecCmd{node_ip = Node.ip, exec_cmd = "/OSM/script/upgrade_pcie.sh -e"};
        end
    end

    return;
end

--------------------------------------------------------------------------------
-- Func Name       :CleanBackDirAfterUpgradeSuccess
-- Description     :流程成功删除BAK目录 仅保存 versions.conf 文件
-- Caution         :
-- Input           :无
-- Output          :无
-- Return          :无
--------------------------------------------------------------------------------
function CleanBackDirAfterUpgradeSuccess()
    -- 在线或离线升级 清除BAK目录
    if(("OnlineUpgrade" ~= g_UpdState.Flow) and ("OfflineUpgrade" ~= g_UpdState.Flow))then
        return;
    end

    -- 升级不成功 不予清除
    if(0 ~= #g_UpdState.FailNodeList) then
        return;
    end

    local cmd = "/OSM/script/os_system_update.sh -L";
    LogText(LL_NOTICE, "Flow success and need to delete back dir.");

    -- 清除集群主上BAK目录
    System(cmd, 3);

    -- 清除非主节点的BAK目录
    for Index, Node in pairs(g_UpdState.NodeList) do
        if (not IsMasterNode(Node.ip)) then
            LogText(LL_NOTICE, "Upgrade success, delete boot_osp_bak: " .. Node.ip);
            ServerSendMsgToAgentExecCmd{node_ip = Node.ip, exec_cmd = "/OSM/script/os_system_update.sh -L"};
        end
    end

    return;
end

--------------------------------------------------------------------------------
-- Func Name       :IsSomeNodeExecUpdSuccess
-- Description     :判断是否存在节点执行升级成功
--------------------------------------------------------------------------------
function IsSomeNodeExecUpdSuccess()
    for _, Node in pairs(g_UpdState.NodeList) do
        if((not g_UpdateFlowsTbl[g_UpdState.Flow].isRbk) and (Node.st == NODE_STATE_SUCC)) then
            LogText(LL_NOTICE, "Some node upgrade succeed.");
            return true;
        end
        if((g_UpdateFlowsTbl[g_UpdState.Flow].isRbk) and (Node.st == NODE_STATE_SUCC_RBK)) then
            LogText(LL_NOTICE, "Some node manual rollback fail.");
            return true;
        end
    end
    LogText(LL_NOTICE, "All node upgrade or manual rollback failed.");
    return false;
end

--------------------------------------------------------------------------------
-- Func Name       :IsAGNeedRebootCluster
-- Description     :判断AG是否需要重启集群，包括离线准备之后重启之前的AG
--------------------------------------------------------------------------------
function IsAGNeedRebootCluster(AtomGroup)
    local RebootClusterAG = {["UpgradePcieSw"]=true, ["UpgradeSystemWhenOfflineUpgrade"]=true,
                             ["RebootWhenOfflineUpgrade"]=true, ["RollbackSystemWhenOfflineUpgrade"]=true,
                             ["RollbackRebootWhenOfflineUpgrade"]=true};


    if(RebootClusterAG[AtomGroup])then
        LogText(LL_NOTICE, "The AtomGroup need reboot cluster.");
        return true;
    end
    LogText(LL_NOTICE, "The AtomGroup no need reboot cluster.");
    return false;
end
--------------------------------------------------------------------------------
-- Func Name       :GetRebootType
-- Description     :获取重启的类型，不需要重启、重启集群、重启失败节点
--------------------------------------------------------------------------------
function GetRebootType()

    -- 初始化阶段失败设置为预处理失败，不进入fault
    if ((g_UpdState.g_FlowStage ~= nil) and (g_UpdState.g_FlowStage.stage == STAGE_INIT_FLOW)) then
        return NO_NEED_REBOOT;
    end
    -- 第一阶段失败设置为预处理失败，不进入fault
    if ((g_UpdState.g_FlowStage ~= nil) and (g_UpdState.g_FlowStage.stage == STAGE_FIRST_FLOW)) then
        return NO_NEED_REBOOT;
    end
    --第三阶段失败进入fault
    if ((g_UpdState.g_FlowStage ~= nil) and (g_UpdState.g_FlowStage.stage == STAGE_THIRD_FLOW)) then
        if (0 ~= #g_UpdState.FailNodeList) then
            LogText(LL_NOTICE, "Some node upgrade fail in the third stage.");
            return REBOOT_FAIL_NODE;
        else
            return NO_NEED_REBOOT;
        end
    end

    --部分成功，部分失败并且没有执行大回退场景下重启失败节点，或者重启集群
    if((0 == g_UpdState.FlowUpdState) and (0 ~= #g_UpdState.FailNodeList)) then
        LogText(LL_NOTICE, "Some node upgrade fail and may need reboot fail node.");

        --在线升级第三批次节点失败不进行大回退, 需要考虑重启失败节点
        if(("OnlineUpgrade" == g_UpdState.Flow) and (1 == g_UpdState.g_FinishAllBatch))then
            LogText(LL_NOTICE, "The last batch upgrade fail and need reboot fail node.");
            return REBOOT_FAIL_NODE;
        end

        --此处需要考虑未回退的场景，所以只需要从upd属性获取
        local AtomGroup;
        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.AtomGroupIndex];
        if(("OfflineUpgrade" == g_UpdState.Flow) and (AtomGroup.IsNeedReboot))then
            --流程执行到最后AG，并且存在成功节点时需要重启失败节点
            if((g_UpdState.AtomGroupIndex == #(g_UpdateFlowsTbl[g_UpdState.Flow].upd)) and (IsSomeNodeExecUpdSuccess()))then
                LogText(LL_NOTICE, "The offline upgrade or rollback fail and need reboot fail node.");
                return REBOOT_FAIL_NODE;
            end
       end
    end

    if(1 == g_UpdState.FlowUpdState) then
        LogText(LL_NOTICE, "rollback and may need reboot node.");
        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.FailRbkAGIndex];
        --离线升级中升级pcie失败之后，回退成功之后系统的当前状态为powering off，需要重启集群，恢复系统状态
        if(("OfflineUpgrade" == g_UpdState.Flow) and ("UpgradePcieSw" == AtomGroup.AtomGroupName)) then
             LogText(LL_NOTICE, "The offline upgrade fail when UpgradePcieSw fail and rollback succeed and need reboot cluster.");
             return REBOOT_CLUSTER;
         end
        --在线、离线升级回退失败时需要进入fault状态
        if ("OnlineUpgrade" == g_UpdState.Flow or "OfflineUpgrade" == g_UpdState.Flow) then
            for _, Node in pairs(g_UpdState.NodeList) do
                if (NODE_STATE_FAIL == Node.st) or (NODE_STATE_TIMEOUT == Node.st) then
                    LogText(LL_NOTICE, "The upgrade rollback fail and need reboot node.");
                    return REBOOT_FAIL_NODE;
                end
            end
            return NO_NEED_REBOOT;
        end
    end

    LogText(LL_NOTICE, "Upgrade finish and no need reboot some node.");
    return NO_NEED_REBOOT;
end

--新版本新增全局变量后,若降级到老版本,老版本残留变量后再升级到新版本会有兼容性问题所以将新增的变量设置成nil
function initSomeState()
    --新版本新增的变量在流程结束后设置成nil, 解决新版本降级老板后残留状态,导致再次升级时状态有问题
    LogText(LL_NOTICE, "Set some state nil when flow finish.");
    g_UpdState.g_MaxBatchNo = nil;
    g_UpdState.g_FinishAllBatch = nil;
    g_UpdState.g_FlowStage = nil;
    g_UpdState.g_isCrossUpgrade = nil;
    SaveState(g_UpdState);
end

-- 有此文件表示fault状态 进fault状态需要产生此文件
function CreateFileForFault()

    LogText(LL_NOTICE, "Create state_version.lua for fault.");
    local fn = "/startup_disk/conf/conf_local/state_version.lua"

    if (nil ~= file_exists(fn)) then
        LogText(LL_NOTICE, "The state_version.lua is exist.");
        return 0;
    end

    local f = io.open(fn, "w+");
    if (nil == f) then
        os.execute("sleep " .. 3);
        f = io.open(fn, "w+");
        if (nil == f) then
            return 1;
        end
    end

    os.execute("chmod 664 " .. fn);
    f:write("{\n");
    f:write("flowname=" .. g_UpdState.Flow .. ",\n");
    f:write("pkgversion=" .. g_UpdState.ParamList.pkgversion .. ",\n");
    f:write("rollversion=" .. g_UpdState.ParamList.rollversion .. ",\n");
    f:write("}\n");
    f:flush(); f:close();
    os.execute("fsync " .. fn);

    -- 同步至其他控制器
    for _, Node in pairs(g_UpdState.NodeList) do
        if (not IsMasterNode(Node.ip)) then
            local file = "/startup_disk/conf/conf_local/state_version.lua";
            local filepath = "/startup_disk/conf/conf_local";

            local iRet=TransSomethingByRealIP{virtual_ip = Node.ip, trans_type = "PUT_FILE", src = file, dst = filepath, timeout=5};
            if(0 ~= iRet) then
                LogText(LL_NOTICE, "Create upgrade state file fail the ip: " .. Node.ip .. "the result: " .. iRet);
                return 1;
            end
        end
    end

    return 0;
end

function DeleteFileForFault()

    LogText(LL_NOTICE, "Delete state_version.lua for fault.");
    local fn = "/startup_disk/conf/conf_local/state_version.lua"

    local rm = os.remove(fn);
    if (nil == rm) then
        LogText(LL_WARN, "Delete upgrade state file fail.");
        return 1;
    end

    for _, Node in pairs(g_UpdState.NodeList) do
        if (not IsMasterNode(Node.ip)) then
            LogText(LL_NOTICE, "Delete state_version.lua for fault.");
            ServerSendMsgToAgentExecCmd{node_ip = Node.ip, exec_cmd = "/OSM/script/upgrade_common.sh -f"};
        end
    end

    return 0;
end

--------------------------------------------------------------------------------
-- Func Name       :RebootFailNodeHandle
-- Description     :升级失败重启集群及重启失败节点处理，新版本只保留重启集群的场景，
--                  重启集群之前需要将流程设置成完成状态
--                  重启失败节点的流程不再调用
-- Input           :无
-- Output          :无
--------------------------------------------------------------------------------
function RebootFailNodeHandle()
    --某些失败场景需要重启集群或者失败节点在此处理
    local reboottype = GetRebootType();
    local rebootNodesInfo;

    local fn = "/startup_disk/conf/conf_local/state_version.lua"
    if ((g_UpdState.Flow == "OnlineUpgrade" or g_UpdState.Flow == "OfflineUpgrade") and file_exists(fn) and IsAllNodeSuccInNodeList() and g_UpdState.g_FlowStage.stage == STAGE_THIRD_FLOW) then
        LogText(LL_NOTICE, "Upgrade success and delete the fault file.");
        DeleteFileForFault();
    end

    --新版本不再重启失败节点，所以将重启失败节点重新赋值成不需要重启
    if(REBOOT_FAIL_NODE == reboottype)then
        CreateFileForFault();
        --fault状态开例测
        NotifyDrvUpdState{upd_state = 2, upd_flowname = g_UpdState.Flow, upd_locationid = "all"};
        local state,bitmap = SetNotifySysState();
        LogText(LL_NOTICE, "---Exec NotifySysStartUpgrade " .. state  .. bitmap );
        local cmd = "/OSM/bin/upgrade_atom -f NOTIFY_SYS_UPDATE_STATE -u " .. state .. " -h " .. g_UpdState.Flow .. " -a " .. bitmap;
        System(cmd, 5);
        LogText(LL_NOTICE, "Upgrade fail and no need auto reboot fail node.");
        --新版本增加了变量降级到老版本，再次升级到新版本的时候需要清空残留的序列化文件中的变量
        initSomeState();
        return FLOW_STATE_FAULT;
    end

    --新版本保留重启集群的情况重启之前需要将状态设置成完成
    if(REBOOT_CLUSTER == reboottype)then
        g_UpdState.FlowState = FLOW_STATE_FINISH;
        PostHandleAfterUpdFinish(g_UpdState.Flow);
        SyncMemeryToDisk();
        --新版本增加了变量降级到老版本，再次升级到新版本的时候需要清空残留的序列化文件中的变量
        initSomeState();
        SaveState(g_UpdState);
        RebootClusterWhenOfflineUpgrade();
        return FLOW_STATE_FINISH;
    elseif(REBOOT_FAIL_NODE == reboottype)then
        rebootNodesInfo = GetRebootNodes();
        if(nil ~= rebootNodesInfo)then
            g_UpdState.g_BackupFlowResult = BACKUP_FLOW_STATE_REBOOT;
            SaveState(g_UpdState);
        end
        PostHandleAfterUpdFinish(g_UpdState.Flow);
        SyncMemeryToDisk();
        RebootFailNode(rebootNodesInfo);
    else
        return;
    end
end


--------------------------------------------------------------------------------
-- Func Name       :FinishFlowHandle
-- Description     :对于在线升级及离线升级,若升级失败大回退结束后,需要重启失败的节点，在线升级,重启第三批次的节点
-- Caution         :
-- Input           :无
-- Output          :无
-- Return          :无
--------------------------------------------------------------------------------
function FinishFlowHandle()

    LogText(LL_NOTICE, "Enter FinishFlowHandle.");
    --当在主控执行的原子组执行时,当前节点列表中不含有此节点,所以需要将正在执行的节点修改成取消
    UpdateNodeState(TERM_FLOW, {}, "Manager");

    --在线离线升级在通知完系统控制升级完成再设置流程状态未完成状态
    if(("OnlineUpgrade" ~= g_UpdState.Flow) and ("OfflineUpgrade" ~= g_UpdState.Flow))then
        g_UpdState.FlowState = FLOW_STATE_FINISH;
    end

    SetBackupFlowInfo();

    --升级过程中新加入节点处理
    NewNodeAddedWhenUpgrade();

    --清除BAK目录
    CleanBackDirAfterUpgradeFail();
    CleanBackDirAfterUpgradeSuccess();
    SaveState(g_UpdState);

    --重启集群及重启失败节点处理流程 重启失败节点依赖某些变量所以在此函数中初始化新增全局变量
    local ret = RebootFailNodeHandle();
    if (FLOW_STATE_FAULT == ret) then
        LogText(LL_NOTICE, "Upgrade failed and create fault file finish flow.");
        g_UpdState.FlowState = FLOW_STATE_FINISH;
        SaveState(g_UpdState);
        SyncMemeryToDisk();
        return;
    elseif (FLOW_STATE_FINISH == ret) then
        return;
    end

    PostHandleAfterUpdFinish(g_UpdState.Flow);
    LogText(LL_NOTICE, "Upgrade finish and no need reboot.");

    --新版本增加了变量降级到老版本，再次升级到新版本的时候需要清空残留的序列化文件中的变量
    initSomeState();

    --在线升级离线升级在通知系统控制升级完成后在设置升级完成
    g_UpdState.FlowState = FLOW_STATE_FINISH;
    SaveState(g_UpdState);
end

--------------------------------------------------------------------------------
-- Func Name       :NewNodeAddedWhenUpgrade
-- Description     :离线升级前如果有节点不在位，升级后要将其作为失败节点处理
-- Caution         :
-- Input           :无
-- Output          :无
-- Return          :无
--------------------------------------------------------------------------------
function NewNodeAddedWhenUpgrade()
    --此场景只针对离线升级
    if ("OfflineUpgrade" ~= g_UpdState.Flow) then
        return;
    end
    LogText(LL_NOTICE, "Enter NewNodeAddedWhenUpgrade.");

    local parm = GetClusterInfo();
    local func = loadstring("return " .. parm); --将字符串转换成table
    local newClusterInfo = func();
    local oldClusterInfo= g_UpdState.g_ClusterInfo;
    local sign = 0;

    for _,newNode in pairs(newClusterInfo) do
        LogText(LL_NOTICE, "nid: " .. newNode.Nid .. "ip: " .. newNode.ip);
        sign = 0;
        for _,node in pairs(oldClusterInfo)do
            if (newNode.Nid == node.Nid) then
                sign = 1;
                break;
            end
        end
        if (sign == 0) then
            table.insert(g_UpdState.FailNodeList, {ip = newNode.ip, type = newNode.tp, state = ATOM_GROUP_EXEC_RET_INIT});
            table.insert(g_UpdState.g_ClusterInfo, {ip = newNode.ip, Nid = newNode.Nid ,type = newNode.tp, state = ATOM_GROUP_EXEC_RET_INIT});
        end
    end
end

function IsNotExistPcie(NodeNum)
     local  retcode;
     --单控或双控设备不存在PCE
     if(NodeNum <=2) then
        return true;
     end
     --多控6U设备不存在PCIE
     local cmd = "/OSM/script/upgrade_pcie.sh -c";
     retcode=System(cmd, 3);
     --retcode ：0：存在PCIE，1：不存在PCIE
     if(retcode == 1) then
        return true
     end

    return false;
end

--------------------------------------------------------------------------------
-- Func Name       :GetStartRbkBatchNo
-- Description     :判断当前失败开始回退的批次是否需要执行回退，不需要时更新成下一个批次
-- Caution         :适用于大回退场景，只有第一批次比较特殊，要判断是否需要执行回退
--                  回退的第二批次要执行所有的AG，不会跳过第二批次
-- Input           :
-- Return          :开始回退的RbkBatchNo
--------------------------------------------------------------------------------
function GetStartRbkBatchNo()
    LogText(LL_NOTICE, "Exec GetStartRbkBatchNo.");
    local RbkBatchNo = g_UpdState.RbkBatchNo + 1;
    local RbkAtomGroupName = g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.AtomGroupIndex].rbk;
    --若当前批次的rbk属性为空，则当前批次不需要回退直接更新成下一个批次
    if("" == RbkAtomGroupName or nil == RbkAtomGroupName) then
        RbkBatchNo = RbkBatchNo + 1;
        LogText(LL_NOTICE, "This batch no need exec rbk and update the RbkBatchNo." .. RbkBatchNo);
        return RbkBatchNo;
    end
    --若当前失败的批次不是第一批次，那么回退就分成多个批次，而回退PCIE只在最后一个批次执行
    if(("UpgradePcieSwRollback" == RbkAtomGroupName) and  (1 ~= g_UpdState.BatchNo) and (1 == RbkBatchNo))then
        RbkBatchNo = RbkBatchNo + 1;
        LogText(LL_NOTICE, "This batch no need rbk pcie and update the RbkBatchNo." .. RbkBatchNo);
        return RbkBatchNo;
    end

    --双控，跳过PCIE原子组
    local Ret = 0;
    local NodeNum = 0;
    Ret, NodeNum = GetUsrConfigNodeNum();
    if(RET_OK ~= Ret)then
        LogText(LL_ERROR, "PcieSwRunAtomGroup::GetUsrConfigNodeNum error");
        NodeNum = 4;
    end

    if(("UpgradePcieSwRollback" == RbkAtomGroupName) and (IsNotExistPcie(NodeNum)))then
        RbkBatchNo = RbkBatchNo + 1;
        LogText(LL_NOTICE, "This is not multi controller no need upgrade PCIE and update the RbkBatchNo: " .. RbkBatchNo);
        return RbkBatchNo;
    end
    LogText(LL_NOTICE, "This batch need rbk pcie and no need update the RbkBatchNo." .. RbkBatchNo);
    return RbkBatchNo;
end


--------------------------------------------------------------------------------
-- Func Name       :GetStartAGIndex
-- Description     :获取执行大回退时从哪个AG开始回退
-- Caution         :适用于大回退场景，只有第一批次比较特殊，需要判断开始回退的AG
--                  回退的第二批次要从第一个AG开始回退
-- Input           :RbkBatchNo：回退的批次
-- Return          :开始回退的AGIndex
--------------------------------------------------------------------------------
function GetStartAGIndex(RbkBatchNo)
    if(0 == g_UpdState.FlowUpdState)then
        LogText(LL_NOTICE, "Upgrade flow need start from the frist atom group.");
        return 0;
    end
    --升级失败后大回退流程，根据配置的rbk属性获取开始回退的AG
    if(1 == RbkBatchNo)then
        local RbkAtomGroupName = g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.AtomGroupIndex].rbk;
        for i = 1, #g_UpdateFlowsTbl[g_UpdState.Flow].rbk do
            if (g_UpdateFlowsTbl[g_UpdState.Flow].rbk[i].AtomGroupName == RbkAtomGroupName) then
                LogText(LL_NOTICE, "Automatic rollback flow need start from the atom group:" .. i);
                return i-1;
            end
        end
    else
        LogText(LL_NOTICE, "Rbk flow need start from the frist atom group.");
        return 0;
    end
end
--------------------------------------------------------------------------------
-- Func Name       : GetAtomGroupIndex
-- Description     : 通过AG 的名字获取其index
--------------------------------------------------------------------------------
function GetAtomGroupIndex(atomgroup)
    local index = nil;
    if( g_UpdState.Flow == "OnlineUpgrade" or g_UpdState.Flow == "OfflineUpgrade" )then
        if(0 == g_UpdState.FlowUpdState)then
            for i = 1,#g_UpdateFlowsTbl[g_UpdState.Flow].upd  do
                if (g_UpdateFlowsTbl[g_UpdState.Flow].upd[i].AtomGroupName == atomgroup) then
                    index = i;
                    LogText(LL_NOTICE, "The node  exec upd atom group index and GetAtomGroupIndex is:" .. index .. "," .. g_UpdState.AtomGroupIndex);
                end
            end
        else
            for i = 1,#g_UpdateFlowsTbl[g_UpdState.Flow].rbk do
                if (g_UpdateFlowsTbl[g_UpdState.Flow].rbk[i].AtomGroupName == atomgroup) then
                    index = i;
                    LogText(LL_NOTICE, "The node  exec rbk atom group index and GetAtomGroupIndex is " .. index .. "," .. g_UpdState.AtomGroupIndex);
                end
            end
        end
    end
    return index;
end
--------------------------------------------------------------------------------
-- Func Name       :IsAllNodeSucc
-- Description     :是否所有节点都执行成功
-- Caution         :
-- Input           :
-- Return          :false 有节点执行失败
--                 :true 所有节点都执行成功
--------------------------------------------------------------------------------
function IsAllNodeSucc()
    local AtomGroup;
    local nodeAtomGroup;
    local nodeAgIndex;

    if( 1 == g_UpdState.g_abNormalRebootFlag )then
        g_UpdState.g_abNormalRebootFlag = 0;
        LogText(LL_NOTICE, "Some node abnormal reboot and set current flow fail.");
        return false;
    end

    --判断当前节点的AGIndex是否与server侧记录的g_UpdState.AtomGroupIndex一致
    --若AGInex大于g_UpdState.AtomGroupIndex说明有异常复位
    --由于专家模式忽略的场景会导致当前节点的AGIndex小于g_UpdState.AtomGroupIndex
    --暂时不考虑异常复位AGIndex小于g_UpdState.AtomGroupIndex的场景
    for _, Node in pairs(g_UpdState.CurNodeList) do
        nodeAtomGroup = g_UpdState.NodeList[ip2int(Node.ip)].ag;
        nodeAgIndex = GetAtomGroupIndex(nodeAtomGroup);
        if( nil ~= nodeAgIndex and g_UpdState.AtomGroupIndex < nodeAgIndex)then
            LogText(LL_NOTICE, "Server abnormal reboot and set current flow fail." .. nodeAgIndex);
            return false;
        end
    end

    if(0 == g_UpdState.FlowUpdState)then
        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.AtomGroupIndex];
    else
        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].rbk[g_UpdState.AtomGroupIndex];
    end
    --目前流程中Manager执行的原子组执行成功后，继续执行后续AG，执行成功时会将MasterList中节点的状态更新成成功
    if("Manager" == AtomGroup.ExcuteNodeType)then
        for _, Node in pairs(g_UpdState.MasterList) do
            if((0 == g_UpdState.FlowUpdState and ((ATOM_GROUP_EXEC_RET_SUCC ~= Node.state and not g_UpdateFlowsTbl[g_UpdState.Flow].isRbk)
                or (ATOM_GROUP_EXEC_RET_SUCC_RBK ~= Node.state and g_UpdateFlowsTbl[g_UpdState.Flow].isRbk)))
                or (1 == g_UpdState.FlowUpdState and ATOM_GROUP_EXEC_RET_SUCC_RBK ~= Node.state)) then
                LogText(LL_NOTICE, "Manager upgrade or rbk fail Node.ip:" .. Node.ip);
                return false;
            end
        end
        LogText(LL_NOTICE, "Manager upgrade or rbk succeed.");
        return true;
    end
    for _, Node in pairs(g_UpdState.CurNodeList) do
        if(((g_UpdState.NodeList[ip2int(Node.ip)]).ag == AtomGroup.AtomGroupName)
            and ((0 == g_UpdState.FlowUpdState and ((ATOM_GROUP_EXEC_RET_SUCC ~= Node.state and not g_UpdateFlowsTbl[g_UpdState.Flow].isRbk)
                or (ATOM_GROUP_EXEC_RET_SUCC_RBK ~= Node.state and g_UpdateFlowsTbl[g_UpdState.Flow].isRbk)))
            or (1 == g_UpdState.FlowUpdState and ATOM_GROUP_EXEC_RET_SUCC_RBK ~= Node.state))) then
            LogText(LL_NOTICE, "Some node upgrade or rbk fail Node.ip:" .. Node.ip);
            return false;
        end
    end
    LogText(LL_NOTICE, "All nodes upgrade or rbk succeed.");
    return true;
end

--------------------------------------------------------------------------------
-- Func Name       :IsAllNodeFail
-- Description     :是否所有节点都执行失败
-- Return          :true  所有节点执行失败
--                 :false 有节点执行成功
--------------------------------------------------------------------------------
function IsAllNodeFail()
    local AtomGroup;
    local FailNum = 0;
    if(0 == g_UpdState.FlowUpdState)then
        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.AtomGroupIndex];
    else
        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].rbk[g_UpdState.AtomGroupIndex];
    end
    --目前流程中Manager执行的原子组, 失败后直接退出,此处可以不加判断，若后续Manager执行的AG失败后需要继续执行后续的流程，则需要再此添加判断条件
    if("Manager" == AtomGroup.ExcuteNodeType)then
        LogText(LL_NOTICE, "Manager upgrade or rbk fail.");
        return true;
    end

    for _, Node in pairs(g_UpdState.CurNodeList) do
        if((0 == g_UpdState.FlowUpdState and ((ATOM_GROUP_EXEC_RET_SUCC ~= Node.state and not g_UpdateFlowsTbl[g_UpdState.Flow].isRbk)
            or (ATOM_GROUP_EXEC_RET_SUCC_RBK ~= Node.state and g_UpdateFlowsTbl[g_UpdState.Flow].isRbk)))
            or (1 == g_UpdState.FlowUpdState and ATOM_GROUP_EXEC_RET_SUCC_RBK ~= Node.state)) then
            FailNum = FailNum + 1;
        end
    end
    if(FailNum == #g_UpdState.CurNodeList)then
        LogText(LL_NOTICE, "All nodes upgrade or rbk fail.");
        return true;
    end

    LogText(LL_NOTICE, "Some node upgrade or rbk succeed.");

    --最后一批先执行所有普通节点的agent准备再执行主控的manager准备，如果此时所有agent准备都失败了就应该算作失败
    LogText(LL_NOTICE, "The failNum is: " .. FailNum .. "AtomGroupName is: " .. AtomGroup.AtomGroupName);
    if((#g_UpdState.CurNodeList == 1 + FailNum) and ("AgentPrepareForOnlineUpgrade" == AtomGroup.AtomGroupName))then
        LogText(LL_NOTICE, "Agent prepare fail when exec the last batch.");
        for _, Node in pairs(g_UpdState.CurNodeList) do
            if (ATOM_GROUP_EXEC_RET_SUCC == Node.state) then
                local ag = GetAtomGroupEx(Node.ip);
                if(ag == "UploadPackageWhenOnlineUpgrade")then
                    LogText(LL_NOTICE, "Agent prepare fail when exec the last batch, terminate the flow.");
                    return true;
                end
            end
        end
    end

    return false;
end

--------------------------------------------------------------------------------
-- Func Name       :GetNextStep
-- Description     :根据当前批次节点执行AG的结果获取下一步骤,此处需要注意，当返回继续执行下一个AG时，需要确保有节点是执行成功状态这样才可以继续后续的流程，因此需要对所有节点都执行失败的做相关判断
-- Return          :EXEC_NEXT_AG       继续执行下一个AG
--                 :EXEC_AUTO_RBK 执行升级失败后的回退
--                 :EXEC_END_FLOW      终止flow
--------------------------------------------------------------------------------
function GetNextStep()
    --第一批次首次执行此函数，需要执行AG
    if(0 == g_UpdState.BatchNo)then
        LogText(LL_NOTICE, "This is the first batch and should exec next atom group.");
        return EXEC_NEXT_AG;
    end

    --在线升级第一批次和第二批次节点发生异常主备倒换后回退
    if(IsAbnormalSwapMaster())then
        LogText(LL_NOTICE, "Abnormal swap master and need rollback system.");
        --上报异常主备倒换原子信息，在界面显示，以示提示由于异常主备倒换导致的回退。
        ReportAbnormalAGInfo{atom_group="AbnormalSwapMaster", node_ip=GetMasterIp(), error_code=3,result=1};
        --专家模式非继续流程，在进入回退前挂起
        if ((g_UpdState.ParamList.mode == MODE_PROFESSIONAL) and (g_UpdState.FlowState == FLOW_STATE_RUNNING)) then
            LogText(LL_NOTICE, "Suspending flow before rollback when abnormal swap master.");
            g_UpdState.FlowState = FLOW_STATE_RBK_SUSPEND;
            SaveState(g_UpdState);
            return EXEC_RBK_SUSPEND;
        end
        g_UpdState.FlowState = FLOW_STATE_RUNNING;
        SaveState(g_UpdState);
        return EXEC_AUTO_RBK;
    end


    --当前批次所有节点执行当前AG成功，则继续执行后续的AG，只在主控执行的AG执行成功后，也继续执行后续的AG
    if(IsAllNodeSucc())then
        LogText(LL_NOTICE, "All node upgrade or rbk succ and exec next atom group.");
        return EXEC_NEXT_AG;
    end

    --对于部分成功部分失败的场景按照升级流程及升级失败后的回退流程分别采用不同的处理方式如下
    --当前的流程为升级流程时的处理
    if(0 == g_UpdState.FlowUpdState)then
        --同步流程, 执行失败时仍然要通知系统控制结果，所以继续AG
        if ("SyncUpgrade" == g_UpdState.Flow)then
            LogText(LL_NOTICE, "SyncUpgrade must callback sys.");
            return EXEC_NEXT_AG;
        end
        --不需要执行大回退的流程，若存在节点执行当前的AG失败，则需要终止flow
        if(not g_UpdateFlowsTbl[g_UpdState.Flow].isFlowRbk)then
            if(not g_UpdateFlowsTbl[g_UpdState.Flow].isRbk or IsAllNodeFail())then
                LogText(LL_NOTICE, "Some node upgrade fail and the flow no need rollback, so terminate flow.");
                return EXEC_END_FLOW;
            else --手动回退时，部分成功部分失败需要继续执行后续的流程
                LogText(LL_NOTICE, "Some node upgrade rollback fail, and need exec next atom group.");
                return EXEC_NEXT_AG;
            end

        --需要大回退的流程根据具体的逻辑判断是执行大回退还是继续执行后续的AG
        else
            if(IsTheFailRatioNeedRbk())then
                --专家模式非继续流程，在进入回退前挂起
                if ((g_UpdState.ParamList.mode == MODE_PROFESSIONAL) and (g_UpdState.FlowState == FLOW_STATE_RUNNING)) then
                    LogText(LL_NOTICE, "Suspending flow before rollback.");
                    g_UpdState.FlowState = FLOW_STATE_RBK_SUSPEND;
                    SaveState(g_UpdState);
                    return EXEC_RBK_SUSPEND;
                end

                --重新执行时恢复流程状态
                g_UpdState.FlowState = FLOW_STATE_RUNNING;
                SaveState(g_UpdState);
                --首先判断当前执行的AG失败后，是否需要执行大回退，若rbk属性为空则不需要执行回退直接终止flow
                local RbkAtomGroupName = g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.AtomGroupIndex].rbk;
                --此处主要针对不需要分批次执行的离线升级，若当前AG失败不需要回退则终止flow,所以对于不需要执行回退的AG失败后，升级状态不会修改成升级失败回退
                if((not g_UpdateFlowsTbl[g_UpdState.Flow].isInBatches) and (nil == RbkAtomGroupName or "" == RbkAtomGroupName))then
                    LogText(LL_NOTICE, "This atom group fail but no need rollback, so terminate flow.");
                    return EXEC_END_FLOW;
                end
                --在线升级直接返回需要回退，回退如何进行放在后面的流程中判断
                LogText(LL_NOTICE, "Some node upgrade fail and need rollback.");
                return EXEC_AUTO_RBK;
            end
            --专家模式非继续流程，在进入升级前挂起
            if ((g_UpdState.ParamList.mode == MODE_PROFESSIONAL) and (g_UpdState.FlowState == FLOW_STATE_RUNNING)) then
                LogText(LL_NOTICE, "Suspending flow before upd.");
                g_UpdState.FlowState = FLOW_STATE_UPD_SUSPEND;
                SaveState(g_UpdState);
                return EXEC_UPD_SUSPEND;
            end
            g_UpdState.FlowState = FLOW_STATE_RUNNING;
            SaveState(g_UpdState);
            --需要分批次执行的流程，若某批次节点部分成功部分失败不回退时，要确保有成功节点可以继续后续的流程
            --不分批次执行的流程，部分成功部分失败不回退时，部分成功是必须的条件所以不需要判断，并且所有节点执行失败时走回退流程
            if(g_UpdateFlowsTbl[g_UpdState.Flow].isInBatches and IsAllNodeFail())then
                LogText(LL_NOTICE, "This batch no need rollback, so terminate flow g_UpdState.BatchNo:" .. g_UpdState.BatchNo);
                return EXEC_END_FLOW;
            end
            LogText(LL_NOTICE, "Some node upgrade fail but no need rollback.");
            return EXEC_NEXT_AG;
        end
    end

    --当前流程为升级失败后回退流程的处理
    --不需要分批次的执行流程，回退过程中若部分成功部分失败需要继续后续的流程，因此要判断是否所有节点执行失败
    --执行离线升级失败后的回退，存在节点失败，不终止flow
    if(not g_UpdateFlowsTbl[g_UpdState.Flow].isInBatches)then
        if(IsAllNodeFail())then
            LogText(LL_NOTICE, "All node offline rollback fail and need terminate flow.");
            return EXEC_END_FLOW;
        end
        LogText(LL_NOTICE, "Some node offline rollback fail and no need terminate flow.");
        return EXEC_NEXT_AG;
    end

    --需要分批次执行的流程，回退的过程中若有节点执行失败，则直接终止flow
    --执行在线升级失败后的回退，存在节点失败，需要终止flow
    if(g_UpdateFlowsTbl[g_UpdState.Flow].isInBatches)then
        LogText(LL_NOTICE, "Some node online rollback fail and need terminate flow.");
        return EXEC_END_FLOW;
    end

    LogText(LL_NOTICE, "GetNextStep return EXEC_NEXT_AG.");
    return EXEC_NEXT_AG;
end

function NotifyMmtWhenSuspend(notify_type)
    local UpdState = 0;
    if(notify_type)then
        UpdState = 1;
    end

    local cmd_NotifyMmt = "/OSM/bin/upgrade_atom -f NOTIFY_MMT_EXPORT" .. " -u " .. UpdState;
    LogText(LL_NOTICE, "Notify mmt: " .. UpdState);
    System(cmd_NotifyMmt, 1);
    return 0;
end

function NotifyAlarmShield(notify_type)
    local UpdState = 0;
    if(notify_type)then
        UpdState = 1;
    end

    local cmd_NotifyAlm = "/OSM/bin/upgrade_atom -f NOTIFY_SHIELD_ALARM" .. " -u " .. UpdState;
    LogText(LL_NOTICE, "Notify alarm: " .. UpdState);
    System(cmd_NotifyAlm, 1);
    return 0;
end


function InitNodeListState()
    for _, Node in pairs(g_UpdState.NodeList) do
        g_UpdState.NodeList[ip2int(Node.ip)].st = NODE_STATE_INIT;
    end
end

--2800单控备份系统软件
function BackupBootOSP()
    LogText(LL_NOTICE, "begin to exec BackupBootOSP.");
    if((GetSizeOfTable(g_UpdState.NodeList) == 1) and (#g_UpdState.g_MasterSlave == 1)
        and ((g_UpdState.Flow == "OfflineUpgrade") or (g_UpdState.Flow == "OnlineUpgrade") or 
        (g_UpdState.Flow == "HotPatchUpgrade") or (g_UpdState.Flow == "HotPatchRollback") or (g_UpdState.Flow == "HotPatchDelete")
        or (g_UpdState.Flow == "AslPatchUpgrade") or (g_UpdState.Flow == "SpecialPatchUpgrade"))) then
        LogText(LL_NOTICE, "It is single controle, need to backup boot_osp.");
        local cmd = "/OSM/script/os_system_update.sh -B";
        return System(cmd, 5);
    end
    if((GetSizeOfTable(g_UpdState.NodeList) == 1) and (#g_UpdState.g_MasterSlave == 1)
        and (g_UpdState.Flow == "DownloadPkg")) then
        LogText(LL_NOTICE, "It is single controle, need to backup disk file to ccdb.");
        local cmd = "/OSM/script/os_system_update.sh -X";
        System(cmd, 2);
    end
    LogText(LL_NOTICE, "It is not single controler or not OfflineUpgrade flow, don't need to backup boot_osp.");
    return 0;
end

--记录升级历史路径到/OSM/ccdb/store/upgrade_path.log
function RecordUpgradePath()
    if((g_UpdState.Flow == "OfflineUpgrade") or (g_UpdState.Flow == "OnlineUpgrade")) then
        LogText(LL_NOTICE, "begin to exec RecordUpgradePath.");
        local cmd = "/OSM/script/os_system_update.sh -A";
        return System(cmd, 1);
    end
    LogText(LL_NOTICE, "It is not OnlineUpgrade or not OfflineUpgrade flow, don't need to record upgrade path.");
    return 0;
end
--------------------------------------------------------------------------------
-- Func Name       :RunUpgradeFlow
-- Description     :更新节点状态及批次信息,驱动升级流程继续
-- Caution         :先执行当前批次节点的下一个AtomGroup，所有AtomGroup执行完成后执行下一个批次
-- Return          :FLOW_DISPATCH_STATE_FINISH    完成flow
--                 :FLOW_DISPATCH_STATE_NORMAL    继续执行flow
--------------------------------------------------------------------------------
function RunUpgradeFlow()
    LogText(LL_NOTICE, "Exec RunUpgradeFlow.");

    local BatchNo = g_UpdState.BatchNo;
    local NodeList = g_UpdState.CurNodeList;
    local AGIndex = g_UpdState.AtomGroupIndex;
    local RbkBatchNo = g_UpdState.RbkBatchNo;

    --检查当前批次节点执行完AG后，是否需要执行大回退
    local NextStep = GetNextStep();

    --下一个步骤为结束flow
    if(EXEC_END_FLOW == NextStep)then
        LogText(LL_NOTICE, "This batch no need exec next atom group and terminate flow.");
        UpdateNodeState(END_BATCH, g_UpdState.CurNodeList, "Manager");
        return FLOW_DISPATCH_STATE_FINISH;

    --下一个步骤为继续执行AG
    elseif(EXEC_NEXT_AG == NextStep)then
        if((0 == BatchNo) or (IsFinishAllAG(AGIndex)))then
            UpdateNodeState(END_BATCH, g_UpdState.CurNodeList, "Manager");
            --开始第二阶段时将所有节点的状态设置成初始化状态
            if(0 == BatchNo)then
                InitNodeListState();
            end
            --根据当前的升级状态更新批次信息, 升级时更新BatchNo, 升级失败后回退更新RbkBatchNo
            if(0 == g_UpdState.FlowUpdState) then
                LogText(LL_NOTICE, "Finished current batch nodes BatchNo:" .. BatchNo);
                BatchNo = g_UpdState.BatchNo + 1;
            else
                LogText(LL_NOTICE, "Finished current batch nodes RbkBatchNo:" .. RbkBatchNo);
                RbkBatchNo = g_UpdState.RbkBatchNo + 1;
            end
        else
            LogText(LL_NOTICE, "This batch need exec the next atom group.");
            return FLOW_DISPATCH_STATE_NORMAL;
        end

    --下一个步骤为执行大回退
    elseif(EXEC_AUTO_RBK == NextStep)then
        UpdateNodeState(END_BATCH, g_UpdState.CurNodeList, "Manager");
        g_UpdState.FlowUpdState = MANAGER_FLOW_STATE_UPG_RBK;
        --若当前批次执行AG失败不需要回退时，更新成下一个批次
        RbkBatchNo = GetStartRbkBatchNo();
        --记录失败开始回退的AGIndex, 后续大回退时判断是否需要执行升级准备
        g_UpdState.FailRbkAGIndex = AGIndex;
        LogText(LL_NOTICE, "Current batch upgrade fail and need start rollback RbkBatchNo: " .. RbkBatchNo);
    elseif (EXEC_UPD_SUSPEND == NextStep) then
        NotifyMmtWhenSuspend(true);
        if (MODE_SHIELD_ALARM == g_UpdState.ParamList.shieldalarmtype) then
	        NotifyAlarmShield(false);              --专家模式悬挂时解除告警屏蔽
	    end
        NotifyDrvUpdState{upd_state = 2, upd_flowname = g_UpdState.Flow, upd_locationid = "all"};--专家模式悬挂的时候要先开启例测
        if (1 == g_UpdState.g_AGPauseFalg) then
            --如果是暂停，则返回暂停状态
            return FLOW_DISPATCH_STATE_UPD_PAUSE;
        end
        return FLOW_DISPATCH_STATE_UPD_SUSPEND;
    elseif (EXEC_RBK_SUSPEND == NextStep) then
        NotifyMmtWhenSuspend(true);
	    if (MODE_SHIELD_ALARM == g_UpdState.ParamList.shieldalarmtype) then
	        NotifyAlarmShield(false);             --专家模式悬挂时解除告警屏蔽
	    end
        NotifyDrvUpdState{upd_state = 2, upd_flowname = g_UpdState.Flow, upd_locationid = "all"};--专家模式悬挂的时候要先开启例测
        if (1 == g_UpdState.g_AGPauseFalg) then
            --如果是暂停，则返回暂停状态
            return FLOW_DISPATCH_STATE_RBK_PAUSE;
        end
        return FLOW_DISPATCH_STATE_RBK_SUSPEND;
    end

    --根据当前的升级状态获取升级或者回退的节点列表
    NodeList = GetNodeListByPlugin(BatchNo, RbkBatchNo);

    --获取下一批次执行的节点为空, 结束流程
    if (nil == NodeList or 0 == #NodeList) then
        LogText(LL_NOTICE, "NodeList is null and terminate flow.");
        return FLOW_DISPATCH_STATE_FINISH;
    end

    --最后一个批次将g_UpdState.BatchNo设置成3兼容老版本
    if(BatchNo == g_UpdState.g_MaxBatchNo)then
        g_UpdState.g_FinishAllBatch = 1;
        LogText(LL_NOTICE, "This is the last batch and set g_UpdState.g_FinishAllBatch." .. g_UpdState.g_FinishAllBatch);
    end
    --当前批次执行结束, 初始化开始执行下一批节点信息
    UpdateNodeState(START_BATCH, NodeList, "Manager");

    --继续执行下一批次 获取开始执行的AGIndex
    g_UpdState.AtomGroupIndex = GetStartAGIndex(RbkBatchNo);
    g_UpdState.BatchNo = BatchNo;
    g_UpdState.RbkBatchNo = RbkBatchNo;
    g_UpdState.CurNodeList = NodeList;
    SaveState(g_UpdState);
    LogText(LL_NOTICE, "RunUpgradeFlow  return FLOW_DISPATCH_STATE_NORMAL.");
    return FLOW_DISPATCH_STATE_NORMAL;
end

--------------------------------------------------------------------------------
-- Func Name       :IsMasterInThisBatch
-- Description     :判断当前批次节点是否包含主控
-- Caution         :适用于在线升级准备跳过MasterPrePare原子组
-- Input           :无
-- Return          :true 当前批次含主节点
--                 :false 当前批次不含主节点
--------------------------------------------------------------------------------
function IsMasterInThisBatch()
    local MasterIp = GetMasterIp();
    for _, CurNode in pairs(g_UpdState.CurNodeList) do
        if((MasterIp == CurNode.ip) and ((CurNode.state == ATOM_GROUP_EXEC_RET_SUCC) or (CurNode.state == ATOM_GROUP_EXEC_RET_INIT))) then
            LogText(LL_NOTICE, "There is master in this batch: " .. g_UpdState.BatchNo);
            return true;
        end
    end
    LogText(LL_NOTICE, "There is no master in this batch: " .. g_UpdState.BatchNo);
    return false;
end

--------------------------------------------------------------------------------
-- Func Name       :IsOnlyMasterLeftIn3rdBatch
-- Description     :判断在线升级当前批次是否只有或者只剩下主节点需要升级
-- Caution         :当前用于判断在线升级当前批次是否需要执行AgentPrepareForOnlineUpgrade
-- Input           :无
-- Return          :true 升级当前批次只有或者只剩下主节点
--                 :false 升级当前批次不只有或者只剩下主节点
--------------------------------------------------------------------------------
function IsOnlyMasterLeftIn3rdBatch()
    --当在线升级当前批次只有一个节点时判断是否是主节点
    if (1 == #g_UpdState.CurNodeList) then
        if (IsMasterNode(g_UpdState.CurNodeList[1].ip)) then
            return true
        else
            return false
        end
    end

    --当在线升级当前批次有多个节点时，判断是否只剩下主节点，其他节点均已升级失败
    for _, Node in pairs(g_UpdState.CurNodeList) do
        if (not IsMasterNode(Node.ip) and ((Node.state == ATOM_GROUP_EXEC_RET_SUCC) or (Node.state == ATOM_GROUP_EXEC_RET_INIT))) then
            return false
        end
    end
    return true
end

--------------------------------------------------------------------------------
-- Func Name       :UpdateAtomGroupIndex
-- Description     :更新执行的AG信息 一般情况下只需要将AG加1, 某些特殊情况下,跳过特定的原子组
-- Caution         :此函数只能用于跳过非最后一个AG的原子组,最后一个AG不能再此处跳
-- Input           :无
-- Return          :AGIndex 要执行的AG对应的下标
--------------------------------------------------------------------------------
function UpdateAtomGroupIndex()
    local AGIndex = g_UpdState.AtomGroupIndex + 1;

    --根据当前的升级状态获取升级AG或者回退AG
    local AtomGroup;
    if(0 == g_UpdState.FlowUpdState)then
        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[AGIndex];
    else
        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].rbk[AGIndex];
    end

    --在线升级只在第一批次升级PCIE，其他批次跳过此原子组
    if(("UpgradePcieSw" == AtomGroup.AtomGroupName) and (1 ~= g_UpdState.BatchNo))then
        LogText(LL_NOTICE, "This batch no need upgrade PCIE: " .. g_UpdState.BatchNo);
        AGIndex = AGIndex + 1;
        return AGIndex;
    end

    local Ret = 0;
    local NodeNum = 0;
    Ret, NodeNum = GetUsrConfigNodeNum();
    if(RET_OK ~= Ret)then
        LogText(LL_ERROR, "PcieSwRunAtomGroup::GetUsrConfigNodeNum error");
        NodeNum = 4;
    end
    if((("UpgradePcieSw" == AtomGroup.AtomGroupName) or ("UpgradePcieSwRollback" == AtomGroup.AtomGroupName)) and (IsNotExistPcie(NodeNum)))then
        LogText(LL_NOTICE, "This is not multi controller no need upgrade PCIE: " .. NodeNum);
        AGIndex = AGIndex + 1;
        return AGIndex;
    end

    --在线升级若当前批次无agent节点，则跳过AgentPrepareForOnlineUpgrade
    if(("AgentPrepareForOnlineUpgrade" == AtomGroup.AtomGroupName) and IsOnlyMasterLeftIn3rdBatch())then
        LogText(LL_NOTICE, "Only master node in this batch: " .. g_UpdState.BatchNo);
        AGIndex = AGIndex + 1;
        return AGIndex;
    end

    --在线升级若当前批次只有普通节点即不含有主节点，则不需要执行ManagerPrepare
    if(("ManagerPrepareForOnlineUpgrade" == AtomGroup.AtomGroupName) and (not IsMasterInThisBatch()))then
        LogText(LL_NOTICE, "There is no master in this batch: " .. g_UpdState.BatchNo);
        AGIndex = AGIndex + 1;
        return AGIndex;
    end

    return AGIndex;
end

--------------------------------------------------------------------------------
-- Func Name       :UpdateTheLastAGIndex
-- Description     :执行AG完成后，判断是否要跳过下一个AG
-- Caution         :此函数不能用于跳过第一个AG
-- Input           :无
-- Return          :AGIndex 要执行的AG对应的下标
--------------------------------------------------------------------------------
function UpdateTheLastAGIndex(AGIndex)

    --根据当前的升级状态获取升级AG或者回退AG
    if(0 == g_UpdState.FlowUpdState and AGIndex == #g_UpdateFlowsTbl[g_UpdState.Flow].upd) or
      (1 == g_UpdState.FlowUpdState and AGIndex == #g_UpdateFlowsTbl[g_UpdState.Flow].rbk) then
        LogText(LL_NOTICE, "This is the last AGInex.");
        return AGIndex;
    end
    local AtomGroup;
    if(0 == g_UpdState.FlowUpdState)then
        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[AGIndex + 1];
    else
        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].rbk[AGIndex + 1];
    end
    --回退PCIE只在最后一个批次执行
    if((1 == g_UpdState.FlowUpdState) and ("UpgradePcieSwRollback" == AtomGroup.AtomGroupName)
        and (1 ~= g_UpdState.BatchNo) and (g_UpdState.RbkBatchNo < g_UpdState.BatchNo))then
        LogText(LL_NOTICE, "This batch no need rbk pcie and return AGIndex: " .. g_UpdState.RbkBatchNo .. AGIndex+1);
        return AGIndex + 1;
    end

    --双控，跳过PCIE原子组
    local Ret = 0;
    local NodeNum = 0;
    Ret, NodeNum = GetUsrConfigNodeNum();
    if(RET_OK ~= Ret)then
        LogText(LL_ERROR, "PcieSwRunAtomGroup::GetUsrConfigNodeNum error");
        NodeNum = 4;
    end
    if((("UpgradePcieSw" == AtomGroup.AtomGroupName) or ("UpgradePcieSwRollback" == AtomGroup.AtomGroupName)) and (IsNotExistPcie(NodeNum)))then
        LogText(LL_NOTICE, "This is not multi controller no need upgrade PCIE: " .. NodeNum);
        AGIndex = AGIndex + 1;
        return AGIndex;
    end

    LogText(LL_NOTICE, "UpdateTheLastAGIndex return AGIndex: " .. AGIndex);
    return AGIndex;
end
--------------------------------------------------------------------------------
-- Func Name       :BuildParamForAtomGroup
-- Description     :为某些特殊的原子组构造参数
-- Caution         :若某个原子组执行时需要再次构造参数，则需要设置原子组的ExcuteNodeType属性值为ALL
-- Input           :无
-- Return          :无
--------------------------------------------------------------------------------
function BuildParamForAtomGroup(AtomGroup)
    LogText(LL_NOTICE, "BuildParamForAtomGroup AGIndex: " .. AtomGroup.AtomGroupName);

    --将集群节点个数传到upd_atom.lua侧，用于前端链路冗余状态检查
    local Ret = 0;
    local NodeNum = 0;
    Ret, NodeNum = GetUsrConfigNodeNum();
    if(RET_OK == Ret) then
        g_UpdState.ParamList.NodeTotalNum = NodeNum;
    else
        g_UpdState.ParamList.NodeTotalNum = #(g_UpdState.g_ClusterInfo);
    end
    -- 构造参数属于第二阶段
    if (g_UpdState.g_FlowStage.stage ~= nil) then
        g_UpdState.ParamList.stage = g_UpdState.g_FlowStage.stage;
    end

    if ("CopyPackage" == AtomGroup.AtomGroupName) then
        local WaitNodes = {};
        for _, Node in pairs(g_UpdState.NodeList) do
            if(string.match(Node.ip, "%d+$") ~= string.match(g_UpdState.ParamList.ip, "%d+$"))then
                table.insert(WaitNodes, Node.ip);
            else
                LogText(LL_NOTICE, "The node (" .. g_UpdState.ParamList.ip .. ") has the pkg.");
            end
        end
        if (g_UpdState.DistributePkgInfo == nil) then
            g_UpdState.DistributePkgInfo = {};
        end
        g_UpdState.DistributePkgInfo.WaitNodes = WaitNodes;
        g_UpdState.DistributePkgInfo.DownloadingNodes = {};
        g_UpdState.DistributePkgInfo.SuccNodes = {};
        g_UpdState.DistributePkgInfo.SuccNodes[1] = g_UpdState.ParamList.ip;
        g_UpdState.DistributePkgInfo.FailNodes = {};
    end

    if ("UploadPackageWhenOnlineRollback" == AtomGroup.AtomGroupName)then
        local FailAGName = g_UpdateFlowsTbl[g_UpdState.Flow].upd[g_UpdState.FailRbkAGIndex].AtomGroupName;
        if((1 == g_UpdState.g_SwapMasterFalg) and (1 == g_UpdState.RbkBatchNo) and ("RebootWhenOnlineUpgrade" == FailAGName))then
            g_UpdState.ParamList.CheckNormal = 1;
            LogText(LL_NOTICE, "Need wait normal.");
        else
            g_UpdState.ParamList.CheckNormal = 0;
            LogText(LL_NOTICE, "No Need wait normal.");
        end
    end

    --最后一批检查多路径失败不需要回退
    if ("SysCompatibleAfterReboot" == AtomGroup.AtomGroupName)then
        local Ret = 0;
        local NodeNum = 0;
        Ret, NodeNum = GetUsrConfigNodeNum();
        g_UpdState.ParamList.RollBackFlag = 1;
        if(RET_OK == Ret)then
            if((1 == g_UpdState.g_FinishAllBatch) or ((2 == g_UpdState.BatchNo) and (NodeNum <= 2)))then
                g_UpdState.ParamList.RollBackFlag = 0;
            end
        end
    end

    --异构支持包同步流程适配,仅仅针对于版本同步流程
    if (("SyncSysAndFirmware" == AtomGroup.AtomGroupName) and ("1" == g_UpdState.ParamList.syncflag))then
        LogText(LL_NOTICE, "Rebuild paramlist in SyncSysAndFirmware");
        local Cmd = "/OSM/script/os_system_update.sh -M ";
        local TmpFile="/tmp/aslver.conf";
        local Ret = System(Cmd, 2);
        local Aslver="";
        if(RET_OK == Ret)then
            local fn = io.open(TmpFile, "r");
            if(nil ~= fn)then
                Aslver = fn:read("*l");
                if(nil == Aslver)then
                    Aslver="";
                end
                io.close(fn);
            end
        end
        g_UpdState.ParamList["aslver"] = Aslver;
        os.remove(TmpFile);
    end

    --oem包同步流程适配
    if (("SyncSysAndFirmware" == AtomGroup.AtomGroupName) and ("1" == g_UpdState.ParamList.syncflag))then
        g_UpdState.ParamList.oemflag = 0;
        local oem_name = "/startup_disk/image/oem.tgz";
        if(nil ~= file_exists(oem_name))then
            g_UpdState.ParamList.oemflag = 1;--1表示存在oem包
        end
        LogText(LL_NOTICE, "oem flag is " .. g_UpdState.ParamList.oemflag);
        g_UpdState.ParamList.diskfileflag = 0;
        local diskfw_name = "/startup_disk/image/boot/update_pkg/diskfirmware/hddfw.conf";
        if(nil ~= file_exists(diskfw_name))then
            g_UpdState.ParamList.diskfileflag = 1;--1表示存在导入的黑榜
        end
        LogText(LL_NOTICE, "hddfw flag is " .. g_UpdState.ParamList.diskfileflag);
    end

    if("ALL" == AtomGroup.ExcuteNodeType)then
        if("SyncUpgrade" == g_UpdState.Flow)then
            DelRescuePkgForSync();
            local SourcIp;
            local IpIndex = 1;
            g_UpdState.ParamList.WholeSyncIP = g_UpdState.ParamList.ip;
            local TmpIPList = SplitIPList(g_UpdState.ParamList.ip, ",");
            --将多数版本的ip参数分配给各个节点，按照执行目的ip：源ip的形式组装例如126.126.126.105:126.126.126.106
            --表示126.126.126.105节点要从ip为126.126.126.106的节点获取目标版本
            for Index, Node in pairs(g_UpdState.CurNodeList) do
                if(1 == Index)then
                    SourcIp = Node.ip .. ":" .. TmpIPList[IpIndex] .. ",";
                else
                    SourcIp = SourcIp .. Node.ip .. ":" .. TmpIPList[IpIndex] .. ",";
                end
                IpIndex = IpIndex + 1;
                if(IpIndex > #TmpIPList) then
                    IpIndex = 1;
                end
            end
            g_UpdState.ParamList.SourcIp = SourcIp;
        elseif("OnlineUpgrade" == g_UpdState.Flow)then
            local nidlist;
            nidlist = GetSuccNodeNid(g_UpdState.CurNodeList);
            LogText(LL_NOTICE, "GetCurBatchSuccNid nidlist: ".. nidlist);
            g_UpdState.ParamList.nidlist = nidlist;
        else
            local nidlist;
            nidlist = GetSuccNodeNid(g_UpdState.NodeList);
            LogText(LL_NOTICE, "GetSuccNodeNid nidlist: ".. nidlist);
            g_UpdState.ParamList.nidlist = nidlist;
        end
    end
end

function RunCopyPackageAG(NodeIp, Result)
    --start进入,对于网管导包节点,上报成功
    if (NodeIp == nil and Result == nil) then
        local Ip = g_UpdState.DistributePkgInfo.SuccNodes[1];
        local Node = g_UpdState.NodeList[ip2int(Ip)];
        if (Node ~= nil) then
            Node.rt = ATOM_GROUP_EXEC_RET_INIT;
            Node.ag = "CopyPackage";
        end
        ReportAtomGroup{flow_id=g_UpdState.FlowId, atom_group="CopyPackage", node_ip=g_UpdState.DistributePkgInfo.SuccNodes[1], result=ATOM_GROUP_EXEC_RET_SUCC};
    end

    --在正在传包节点列表中查找上报节点,并根据节点成功失败移到成功节点列表或者失败节点列表中; start进入,正在传包节点列表为空, 不会遍历此表
    for Index, DowningNodeIp in pairs(g_UpdState.DistributePkgInfo.DownloadingNodes) do
        if (DowningNodeIp == NodeIp) then
            if (Result == ATOM_GROUP_EXEC_RET_SUCC) then
                table.insert(g_UpdState.DistributePkgInfo.SuccNodes, DowningNodeIp); --节点上报成功
            else
                table.insert(g_UpdState.DistributePkgInfo.FailNodes, DowningNodeIp); --节点上报失败
            end
            table.remove(g_UpdState.DistributePkgInfo.DownloadingNodes, Index);
            break;
        end
    end

    --若正在传包节点列表为空，则本批次传包完成
    if (g_UpdState.DistributePkgInfo.DownloadingNodes ~= nil and #g_UpdState.DistributePkgInfo.DownloadingNodes ~= 0) then
        LogText(LL_NOTICE, "Not finished to distribute pkg, continue.");
        return;
    end

    --本批次完成时,若有失败节点,则停止传包,进行如下的处理: 1.待传包节点移到失败节点列表并上报成功; 2.成功节点清除/OSM/update目录并移到失败节点列表
    if (g_UpdState.DistributePkgInfo.FailNodes ~= nil and #g_UpdState.DistributePkgInfo.FailNodes ~= 0) then
        --1.待传包节点移到失败节点列表并上报成功
        --上报成功的原因: 当传包失败时, toolkit会解析第一个失败原子的错误码并进行显示.
        --如果上报失败, 就会得到的原子错误码是(.3), toolkit有可能解析到第一个失败原子的错误码就是(.3), 造成显示不当.
        --上报成功, 则会正确的解析真正失败原子的错误码, 进行正确的实现.
        for _, WaitNodeIp in pairs(g_UpdState.DistributePkgInfo.WaitNodes) do
            local Node = g_UpdState.NodeList[ip2int(WaitNodeIp)];
            if (Node ~= nil) then
                Node.rt = ATOM_GROUP_EXEC_RET_INIT;
                Node.ag = "CopyPackage";
            end
            ReportAtomGroup{flow_id=g_UpdState.FlowId, atom_group="CopyPackage", node_ip=WaitNodeIp, result=ATOM_GROUP_EXEC_RET_SUCC};
            table.insert(g_UpdState.DistributePkgInfo.FailNodes, WaitNodeIp);
        end
        g_UpdState.DistributePkgInfo.WaitNodes = {};

        --2.成功节点清除/OSM/update目录
        for _, SuccNodeIp in pairs(g_UpdState.DistributePkgInfo.SuccNodes) do
            ServerSendMsgToAgentExecCmd{node_ip = SuccNodeIp, exec_cmd = "rm -rf /OSM/update/*"};
            table.insert(g_UpdState.DistributePkgInfo.FailNodes, SuccNodeIp);
        end
        g_UpdState.DistributePkgInfo.SuccNodes = {};
        return;
    end

    --若待传包节点列表为空,则传包完成
    if (g_UpdState.DistributePkgInfo.WaitNodes == nil or #g_UpdState.DistributePkgInfo.WaitNodes == 0) then
        LogText(LL_NOTICE, "Finished to distribute pkg.");
        return;
    end

    --以成功节点列表为传包源节点,从待传包节点列表中取出节点,下发传包消息,并移到正在传包节点列表
    local AtomGroup = g_UpdateFlowsTbl["DownloadPkg"].upd[2];
    for _, SuccNodeIp in pairs(g_UpdState.DistributePkgInfo.SuccNodes) do
        g_UpdState.ParamList.ip = SuccNodeIp;
        local NodeIpToDown = g_UpdState.DistributePkgInfo.WaitNodes[1];
        for _, Node in pairs(g_UpdState.CurNodeList) do
            if((Node.ip == NodeIpToDown) and ((Node.state == ATOM_GROUP_EXEC_RET_INIT) or (Node.state == ATOM_GROUP_EXEC_RET_SUCC))) then
                Node.state = ATOM_GROUP_EXEC_RET_INIT;
                NodeExecAtomGroup(Node, AtomGroup, Table2String(g_UpdState.ParamList));
                break;
            end
        end
        table.insert(g_UpdState.DistributePkgInfo.DownloadingNodes, NodeIpToDown);
        table.remove(g_UpdState.DistributePkgInfo.WaitNodes, 1);
    end
end

function notifyNodesExeAG(AtomGroup, AGIndex)
    --传包AG
    if ("CopyPackage" == AtomGroup.AtomGroupName) then
        RunCopyPackageAG(nil, nil);
        return;
    end

    --Manager执行属性的AG
    if ("Manager" == AtomGroup.ExcuteNodeType) then
        ManagerRunAtomGroup(AtomGroup);
        return;
    end

    for _, Node in pairs(g_UpdState.CurNodeList) do
        if("AgentPrepareForOnlineUpgrade" == AtomGroup.AtomGroupName)then
            AgentRunAtomGroup(Node, AtomGroup);
        elseif("ManagerPrepareForOnlineUpgrade" == AtomGroup.AtomGroupName)then
            MasterRunAtomGroup(Node, AtomGroup);
            LogText(LL_NOTICE, "swap master soon, sleep to wait sync state file.");
            g_UpdState.AtomGroupIndex = AGIndex;
            SaveState(g_UpdState);
            os.execute("sleep " .. 45);            --等待状态文件同步完成
            LogText(LL_NOTICE, "swap master soon, sleep end.");
        elseif((not g_UpdateFlowsTbl[g_UpdState.Flow].isRbk) and (0 == g_UpdState.FlowUpdState))then
            if(Node.state == ATOM_GROUP_EXEC_RET_INIT ) or (Node.state == ATOM_GROUP_EXEC_RET_SUCC) then
                Node.state = ATOM_GROUP_EXEC_RET_INIT; --重置节点执行状态
                NodeExecAtomGroup(Node, AtomGroup, Table2String(g_UpdState.ParamList));
            end
        elseif((g_UpdateFlowsTbl[g_UpdState.Flow].isRbk) or (1 == g_UpdState.FlowUpdState))then
            if(Node.state == ATOM_GROUP_EXEC_RET_INIT ) or (Node.state == ATOM_GROUP_EXEC_RET_SUCC_RBK) then
                Node.state = ATOM_GROUP_EXEC_RET_INIT; --重置节点执行状态
                NodeExecAtomGroup(Node, AtomGroup, Table2String(g_UpdState.ParamList));
            end
        end
    end
end

function IsAllNodeSuccInNodeList()

    for _, Node in pairs(g_UpdState.NodeList) do
        if (Node.st ~= NODE_STATE_SUCC) then
            return false;
        end
    end

    return true;
end

function PreUpdFlowHandle()

    g_UpdState.g_FlowStage.batch = g_UpdState.g_FlowStage.batch + 1;

    if (g_UpdState.g_FlowStage.batch >= STAGE_SECONE_BATCH) then
        if (IsNodeFailInCurNodeList() == true) then
            if ((g_UpdState.ParamList.mode == MODE_PROFESSIONAL) and (g_UpdState.FlowState == FLOW_STATE_RUNNING)) then
                g_UpdState.FlowState = FLOW_STATE_END_SUSPEND;
                LogText(LL_NOTICE, "Some nodes failed in first stage, suspend in professional mode.");
                NotifyMmtWhenSuspend(true);
                if (MODE_SHIELD_ALARM == g_UpdState.ParamList.shieldalarmtype) then
                    NotifyAlarmShield(false);                --专家模式暂停的时候解除告警屏蔽
                end
                return FLOW_DISPATCH_STATE_END_SUSPEND;
            end
            UpdateNodeState(END_BATCH, g_UpdState.CurNodeList, "Manager");
            FinishFlowHandle();
            LogText(LL_NOTICE, "Some nodes failed in first stage, finish the flow.");
            return FLOW_DISPATCH_STATE_FINISH;
        end

        g_UpdState.g_FlowStage.stage = STAGE_SECOND_FLOW;
        g_UpdState.g_FlowStage.batch = STAGE_INIT_BATCH;
        LogText(LL_NOTICE, "All nodes successed in first stage, go to next stage.");
        return FLOW_DISPATCH_STATE_NEXT;
    end

    g_UpdState.CurNodeList = GetNodeListForFirstStage();
    UpdateNodeState(START_BATCH, g_UpdState.CurNodeList, "Manager");

    local AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].preupd[1];
    g_UpdState.ParamList.stage = STAGE_FIRST_FLOW;

    for _, Node in pairs(g_UpdState.CurNodeList) do
        NodeExecAtomGroup(Node, AtomGroup, Table2String(g_UpdState.ParamList));
    end

    LogText(LL_NOTICE, "Notify all nodes to execute first stage successed.");
    return FLOW_DISPATCH_STATE_NORMAL;
end

function IsNodeFailInCurNodeList()

    for _, Node in pairs(g_UpdState.CurNodeList) do
        if (Node.state ~= ATOM_GROUP_EXEC_RET_SUCC) then
            return true;
        end
    end

    return false;
end

function IsOddNid(ip)

    local tmpNid = GetNidByIp(ip);

    if ((tmpNid ~= nil) and (tmpNid % 2 == 1)) then
        return true;
    end

    return false;
end

function IsEvenNid(ip)

    local tmpNid = GetNidByIp(ip);

    if ((tmpNid ~= nil) and (tmpNid % 2 == 0)) then
        return true;
    end

    return false;
end

function GetNodeListForFirstStage()

    local NodeListInBatch = {};

    for _, Node in pairs(g_UpdState.NodeList) do
        table.insert(NodeListInBatch, {ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 0});
    end

    return NodeListInBatch;
end

function GetNodeListForThirdStage(batch)

    local NodeListInBatch = {};
    local func;

    if (batch == STAGE_FIRST_BATCH) then
        func = IsEvenNid;
    elseif (batch == STAGE_SECONE_BATCH) then
        func = IsOddNid;
    else
        return nil;
    end

    for _, Node in pairs(g_UpdState.NodeList) do
        if (func(Node.ip) == true) then
            table.insert(NodeListInBatch, {ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 0});
        end
    end

    return NodeListInBatch;
end

function NeedRunWhatForThirdStage(agIndex, batch, nodeList)

    -- 本意是过滤重启失败节点后继续postupd的场景
    -- 四个节点同时下发AG 四个节点全部执行完该AG会再次进入此步 判断是否有节点执行该AG失败
    if (IsNodeFailInCurNodeList() == true and batch ~= STAGE_INIT_BATCH) then
        if ((g_UpdState.ParamList.mode == MODE_PROFESSIONAL) and (g_UpdState.FlowState == FLOW_STATE_RUNNING)) then
            g_UpdState.FlowState = FLOW_STATE_END_SUSPEND;
            LogText(LL_NOTICE, "Some nodes failed in third stage, suspend in professional mode.");
            NotifyMmtWhenSuspend(true);
	    if (MODE_SHIELD_ALARM == g_UpdState.ParamList.shieldalarmtype) then
                NotifyAlarmShield(false);                --专家模式暂停的时候解除告警屏蔽
            end
            return FLOW_DISPATCH_STATE_END_SUSPEND;
        end
        UpdateNodeState(END_BATCH, nodeList, "Manager");
        FinishFlowHandle();
        LogText(LL_NOTICE, "Some nodes failed in third stage, finish the flow.");
        return FLOW_DISPATCH_STATE_FINISH;
    end

    -- 初始批次或本批次AG执行完开启下一批次
    if (batch == STAGE_INIT_BATCH or agIndex >= #g_UpdateFlowsTbl[g_UpdState.Flow].postupd) then
        UpdateNodeState(END_BATCH, nodeList, "Manager");

        --开始第三阶段时将所有节点的状态设置成初始化状态
        if(STAGE_INIT_BATCH == batch)then
            InitNodeListState();
        end

        batch = batch + 1; agIndex = 0;
        if (batch == THIRD_STAGE_MAX_BATCH) then
            LogText(LL_NOTICE, "All batch nodes successed in third stage.");
            return FLOW_DISPATCH_STATE_NEXT;
        end

        nodeList = GetNodeListForThirdStage(batch);
        if ((nil == nodeList) or (0 == #nodeList)) then
            LogText(LL_NOTICE, "Get nodelist for third stage is null, get next batch.");
            batch = batch + 1; agIndex = 0;
            if (batch == THIRD_STAGE_MAX_BATCH) then
                LogText(LL_NOTICE, "All batch nodes successed in third stage.");
                return FLOW_DISPATCH_STATE_NEXT;
            end
            nodeList = GetNodeListForThirdStage(batch);
        end
        UpdateNodeState(START_BATCH, nodeList, "Manager");
    end

    agIndex = agIndex + 1;

    return FLOW_DISPATCH_STATE_NORMAL, batch, agIndex, nodeList;
end

function GetUpdType()
    local updType = 2;

    if (g_UpdState.Flow == "OnlineUpgrade") then
        updType = 0;
    elseif (g_UpdState.Flow == "OfflineUpgrade") then
        updType = 1;
    end

    return updType;
end

-- 多批次 多AG
function PostUpdFlowHandle()

    local ret;
    local agIndex  = g_UpdState.g_FlowStage.agindex;
    local batch    = g_UpdState.g_FlowStage.batch;
    local nodeList = g_UpdState.CurNodeList;

    ret, batch, agIndex, nodeList = NeedRunWhatForThirdStage(agIndex, batch, nodeList);
    if (ret ~= FLOW_DISPATCH_STATE_NORMAL) then
        return ret;
    end

    -- 由于重启后驱动升级标志丢失，再单独通知一次
    NotifyDrvUpdState{upd_state = 1, upd_flowname = g_UpdState.Flow, upd_locationid = "all"};

    local AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].postupd[agIndex];

    g_UpdState.ParamList.stage = STAGE_THIRD_FLOW;
    g_UpdState.ParamList.updType = GetUpdType();

    for _, Node in pairs(nodeList) do
        NodeExecAtomGroup(Node, AtomGroup, Table2String(g_UpdState.ParamList));
    end

    g_UpdState.g_FlowStage.agindex = agIndex;
    g_UpdState.g_FlowStage.batch   = batch;
    g_UpdState.CurNodeList         = nodeList;

    LogText(LL_NOTICE, "Notify nodes to execute third stage successed.");
    return FLOW_DISPATCH_STATE_NORMAL;
end

function IsEnableStageFlow()

    if (g_UpdState.g_FlowStage.stage == STAGE_NOT_ENABLE) then
        return false;
    end

    return true;
end

function IsPreUpgradeStage()

    if (g_UpdState.Flow ~= "OnlineUpgrade" and g_UpdState.Flow ~= "OfflineUpgrade") then
        LogText(LL_NOTICE, "Do not execute first stage, flow is " .. g_UpdState.Flow);
        return false;
    end

    if (g_UpdState.g_FlowStage.stage ~= STAGE_FIRST_FLOW) then
        LogText(LL_NOTICE, "Do not execute first stage, stage is " .. g_UpdState.g_FlowStage.stage);
        return false;
    end

    return true;
end

function IsPostUpgradeStage()

    if (g_UpdState.Flow ~= "OnlineUpgrade" and g_UpdState.Flow ~= "OfflineUpgrade") then
        LogText(LL_NOTICE, "Do not execute third stage, flow is " .. g_UpdState.Flow);
        return false;
    end

    if (g_UpdState.g_FlowStage.stage ~= STAGE_THIRD_FLOW) then
        LogText(LL_NOTICE, "Do not execute third stage, stage is " .. g_UpdState.g_FlowStage.stage);
        return false;
    end

    -- 从V3R2版本升级上来 此标记会置1 不需执行第三阶段 进度需要刷成100
    if (g_UpdState.g_isCrossUpgrade ~= 0) then
        if (g_UpdState.g_isCrossUpgrade ~= nil) then
            LogText(LL_NOTICE, "The cross upgrade flag is " .. g_UpdState.g_isCrossUpgrade);
        end
        for _, Node in pairs(g_UpdState.NodeList) do
            Node.pr = 100;
        end
        LogText(LL_NOTICE, "Upgrade from V3R2 version, do not need to execute third stage.");
        return false;
    end

    return true;
end

function RunPreStageHandle()

    SaveState(g_UpdState);
    LogText(LL_NOTICE, "Run first stage.");

    if (g_UpdState.g_FlowStage.stage == STAGE_INIT_FLOW) then
        g_UpdState.g_FlowStage.stage = STAGE_FIRST_FLOW;
    end

    if (IsPreUpgradeStage() == false) then
        LogText(LL_NOTICE, "Do not need to execute the first stage.");
        return FLOW_DISPATCH_STATE_NEXT;
    end

    return PreUpdFlowHandle();
end

function RunPostStageHandle()

    SaveState(g_UpdState);
    LogText(LL_NOTICE, "Run third stage.");

    if (IsPostUpgradeStage() == false) then
        LogText(LL_NOTICE, "Do not need to execute the third stage.");
        return FLOW_DISPATCH_STATE_NEXT;
    end

    return PostUpdFlowHandle();
end

function RunUpgradeSystemHandle()

    SaveState(g_UpdState);
    LogText(LL_NOTICE, "Run second stage.");

    -- 若当前正在执行第三阶段则跳过此步骤
    if (STAGE_THIRD_FLOW == g_UpdState.g_FlowStage.stage) then
        LogText(LL_NOTICE, "Do not need to execute second stage, go to next stage.");
        return FLOW_DISPATCH_STATE_NEXT;
    end

    --当前的故障修复模式直接跳过第二阶段的升级
    local fn = "/startup_disk/conf/conf_local/state_version.lua"
    if (true == file_exists(fn) and (g_UpdState.Flow == "OnlineUpgrade" or g_UpdState.Flow == "OfflineUpgrade")) then
        LogText(LL_NOTICE, "The state_version.lua is exist, go to next state.");
        g_UpdState.g_FlowStage.stage = STAGE_THIRD_FLOW;
        g_UpdState.g_FlowStage.batch = STAGE_INIT_BATCH;
        return FLOW_DISPATCH_STATE_NEXT;
    end

    --更新批次信息、节点状态、判断是否回退
    local ret = RunUpgradeFlow();
    if(ret ~= FLOW_DISPATCH_STATE_NORMAL) then
        LogText(LL_NOTICE, "Something happend in second stage, ret " .. ret);
        return ret;
    end

    --更新执行的AGIndex信息
    local AGIndex = UpdateAtomGroupIndex();
    local AtomGroup;

    --根据当前的升级状态获取升级AG或者回退AG
    if(0 == g_UpdState.FlowUpdState)then
        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[AGIndex];
    else
        AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].rbk[AGIndex];
    end

    --特殊的AG需要再一次组装参数信息
    BuildParamForAtomGroup(AtomGroup);

    --通知节点执行AG
    notifyNodesExeAG(AtomGroup, AGIndex);

    ----更新保存的AGIndex
    g_UpdState.AtomGroupIndex = AGIndex;
    SaveState(g_UpdState);
    LogText(LL_NOTICE, "Notify nodes to execute second stage successed.");
    return FLOW_DISPATCH_STATE_NORMAL;
end

function ContinueRunThirdStage()

    if (IsEnableStageFlow() == false) then
        LogText(LL_NOTICE, "Not enable the three stage flow, go to next stage.");
        return FLOW_DISPATCH_STATE_NEXT;
    end

    g_UpdState.g_FlowStage.stage = STAGE_THIRD_FLOW;
    g_UpdState.g_FlowStage.batch = STAGE_INIT_BATCH;
    LogText(LL_NOTICE, "It is going to run the third stage flow.");
    return FLOW_DISPATCH_STATE_NEXT;
end

function JudgeIsContinueByRet(ret)

    if (ret == FLOW_DISPATCH_STATE_FINISH) then
        if (IsAllNodeSuccInNodeList()) then
            LogText(LL_NOTICE, "All nodes successed in second stage, go to next stage.");
            return ContinueRunThirdStage();
        else
            FinishFlowHandle();
            LogText(LL_NOTICE, "Some nodes failed in second stage, finish the flow.");
            return FLOW_DISPATCH_STATE_FINISH;
        end
    end

    return ret;
end

function L_ExecThirdStageWhenRbootFailNode(Flow, NodeList, ParamList)

    LogText(LL_NOTICE, "Reboot failed nodes successed, go to third stage.");
    local isCrossUpgrade = g_UpdState.g_isCrossUpgrade;

    InitExecFlowInfo(g_UpdState.FlowId + 16, Flow, NodeList, ParamList);

    -- 初始化后 恢复标记至g_isCrossUpgrade
    if (isCrossUpgrade ~= nil) then
        g_UpdState.g_isCrossUpgrade = isCrossUpgrade;
    end

    local ret = PreHandleBeforeUpdStart(Flow);
    if (RET_OK ~= ret) then
        LogText(LL_NOTICE, "Fail to start third stage for reboot fail node, terminate flow: " .. g_UpdState.Flow .. " with flow id: " .. g_UpdState.FlowId);
        FinishFlowHandle();
        return FLOW_DISPATCH_STATE_FINISH;
    end

    g_UpdState.g_FlowStage.stage = STAGE_THIRD_FLOW;
    g_UpdState.g_FlowStage.batch = STAGE_INIT_BATCH;
    g_UpdState.g_FlowStage.agindex = 0;

    -- 刷新进度
    for _, Node in pairs(g_UpdState.NodeList) do
        if (90 - Node.pr > 0) then
            Node.pr = 90;
        end
    end

    SaveState(g_UpdState);
    return RunPostStageHandle();
end

--------------------------------------------------------------------------------
-- Func Name       :RunFlowFunc
-- Description     :各个flow对应的入口函数
-- Caution         :当前只是区分了Manager执行属性，其他的特殊处理都是在线升级准备相关的
-- SR Correlation  :SR-0000469164 升级流程和回退策略改进 20150306
-- Input           :无
-- Return          :无
--------------------------------------------------------------------------------
function RunFlowFunc()
    LogText(LL_NOTICE, "Exec RunFlowFunc.");

    -- 第一阶段 上传固件
    local ret = RunPreStageHandle();
    if (ret ~= FLOW_DISPATCH_STATE_NEXT) then
        LogText(LL_NOTICE, "Run first stage handle return " .. ret);
        SaveState(g_UpdState);
        return ret;
    end

    -- 第二阶段 升级大系统 接口卡 控制框
    ret = RunUpgradeSystemHandle();
    if (JudgeIsContinueByRet(ret) ~= FLOW_DISPATCH_STATE_NEXT) then
        LogText(LL_NOTICE, "Run second stage handle return " .. ret);
        SaveState(g_UpdState);
        return ret;
    end

    -- 第三阶段 升级剩余固件(大卡 BBU 硬盘框)
    ret = RunPostStageHandle();
    if (ret ~= FLOW_DISPATCH_STATE_NEXT) then
        LogText(LL_NOTICE, "Run third stage handle return " .. ret);
        SaveState(g_UpdState);
        return ret;
    end

    SaveState(g_UpdState);
    FinishFlowHandle();
    LogText(LL_NOTICE, "All stages handle successed.");
    return FLOW_DISPATCH_STATE_FINISH;
end

--------------------------------------------------------------------------------
-- Func Name       :GetMasterIp
-- Description     :获取主控的Ip地址
-- Caution         :当发生主备倒换后会实时更新g_MasterSlave,所以要获取主控相关的信息都要以g_MasterSlave为主
-- Input           :无
-- Return          :返回主控的Ip
--------------------------------------------------------------------------------
function GetMasterIp()
    local MasterIp = nil;
    for _, Node in pairs(g_UpdState.g_MasterSlave) do
        if(Node.IsMaster == "1") then
            MasterIp = Node.ip;
            LogText(LL_NOTICE, "Master IP is:"..MasterIp);
            break;
        end
    end
    return MasterIp;
end
--------------------------------------------------------------------------------
-- Func Name       :GetMasterNid
-- Description     :获取主控的Nid
-- Caution         :当发生主备倒换后会实时更新g_MasterSlave,所以要获取主控相关的信息都要以g_MasterSlave为主
-- Input           :无
-- Return          :返回主控的Nid
--------------------------------------------------------------------------------
function GetMasterNid()
    local MasterNid = nil;
    for _, Node in pairs(g_UpdState.g_MasterSlave) do
        if(Node.IsMaster == "1") then
            MasterNid = Node.Nid;
            LogText(LL_NOTICE, "Master Nid is:".. MasterNid);
            break;
        end
    end
    return MasterNid;
end
--------------------------------------------------------------------------------
-- Func Name       :AgentRunAtomGroup
-- Description     :只在非主节点控执行的某个AG
-- Caution         :当前主要用于在线升级准备的特殊处理，执行此原子组要保证当前批
--                  次节点包含非主控且非主节点的状态为初始化状态或者执行成功状态
-- Input           :Node 要执行的节点  AtomGroup 当前要执行的原子组
-- Return          :无
--------------------------------------------------------------------------------
function AgentRunAtomGroup(Node, AtomGroup)
    LogText(LL_NOTICE, "AgentRunAtomGroup.Node.ip" .. Node.ip .. AtomGroup.AtomGroupName);
    if(not IsMasterNode(Node.ip))then
        if (Node.state == ATOM_GROUP_EXEC_RET_INIT ) or (Node.state == ATOM_GROUP_EXEC_RET_SUCC) then
            Node.state = ATOM_GROUP_EXEC_RET_INIT; --重置节点执行状态
            NodeExecAtomGroup(Node, AtomGroup, Table2String(g_UpdState.ParamList));
        end
    end
end

--------------------------------------------------------------------------------
-- Func Name       :MasterRunAtomGroup
-- Description     :只在主控执行某个AG
-- Caution         :当前主要用于在线升级准备的特殊处理，执行此原子组要保证当前批
--                  次节点包含主控且主控的状态为初始化状态或者执行成功状态
-- Input           :Node 要执行的节点  AtomGroup 当前要执行的原子组
-- Return          :无
--------------------------------------------------------------------------------
function MasterRunAtomGroup(Node, AtomGroup)
    LogText(LL_NOTICE, "MasterRunAtomGroup." .. Node.ip .. AtomGroup.AtomGroupName);
    if(IsMasterNode(Node.ip))then
        if (Node.state == ATOM_GROUP_EXEC_RET_INIT ) or (Node.state == ATOM_GROUP_EXEC_RET_SUCC) then
            Node.state = ATOM_GROUP_EXEC_RET_INIT; --重置节点执行状态
            NodeExecAtomGroup(Node, AtomGroup, Table2String(g_UpdState.ParamList));
        end
    end
end

--------------------------------------------------------------------------------
-- Func Name       :IsRbkFail
-- Description     :判断当前批次是否有节点回退失败
-- Caution         :当前主要用于热补丁升级失败后执行大回退后，判断是否有节点回退失败
-- Input           :无
-- Return          :true 有节点执行失败  false 所有节点回退成功
--------------------------------------------------------------------------------
function IsRbkFail()
    local IsFail = false;
    for _, CurNode in pairs(g_UpdState.CurNodeList) do
        if (CurNode.state ~= ATOM_GROUP_EXEC_RET_INIT and CurNode.state ~= ATOM_GROUP_EXEC_RET_SUCC_RBK) then --未开始执行
            IsFail = true;
        end
    end

    return IsFail;
end

function GetSCHotNodeListByPlugin(BatchNo)
    LogText(LL_NOTICE, "This is single controller.");
    LogText(LL_NOTICE, "GetSCHotNodeListByPlugin, BatchNo: " .. BatchNo);

    local FirstBatch = {};

    if(BatchNo == 1)then
        --单控第一批为所有的节点，其余为NIL
        for _, Node in pairs(g_UpdState.NodeList) do
            table.insert(FirstBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 1});
            g_UpdState.NodeList[ip2int(Node.ip)].batchno = 1;
        end
        SaveState(g_UpdState);
        return FirstBatch;
    else
        LogText(LL_NOTICE, "GetSCHotNodeListByPlugin:Batch Number is invalid." .. BatchNo);
        return nil;
    end
end

function GetDCHotNodeListByPlugin(BatchNo)
    LogText(LL_NOTICE, "This is double controller.");
    LogText(LL_NOTICE, "GetDCHotNodeListByPlugin, BatchNo: " .. BatchNo);

    local FirstBatch = {};
    local SecondBatch = {};

    if(BatchNo == 1)then
        for _, Node in pairs(g_UpdState.NodeList) do
            g_UpdState.NodeList[ip2int(Node.ip)].batchno = 0;
        end
        --主控重启后，新主恢复时重新保存g_MasterSlave中不包含重启节点，若后续重启节点已经恢复，暂时没有再次更新g_MasterSlave
        if(#g_UpdState.g_MasterSlave == 2)then
            SaveState(g_UpdState);
            for _, Node in pairs(g_UpdState.g_MasterSlave) do
                if(Node.IsMaster == "0") then ---双控先升级备再升级主
                    table.insert(FirstBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 1});
                    g_UpdState.NodeList[ip2int(Node.ip)].batchno = 1;
                end
            end
        else

            for _, Node in pairs(g_UpdState.g_MasterSlave) do
                if(Node.IsMaster == "1") then ---双控先升级备再升级主
                    table.insert(FirstBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 1});
                    g_UpdState.NodeList[ip2int(Node.ip)].batchno = 1;
                end
            end
        end

        SaveState(g_UpdState);
        return FirstBatch;
    end

    ---双控获取第二批次，不在第一批次的节点为第二批次节点
    if(2 == BatchNo)then
        for _, Node in pairs(g_UpdState.g_ClusterInfo) do
            if (IsNotInFirstBatch(Node.ip)) then ---
                LogText(LL_NOTICE, "Insert SecondBatch node:" .. Node.Nid);
                table.insert(SecondBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 2});
                g_UpdState.NodeList[ip2int(Node.ip)].batchno = 2;
            end
        end
        SaveState(g_UpdState);
        return SecondBatch;
    end

    ---双控情况，最多执行2个批次
    if(3 == BatchNo) then
        LogText(LL_NOTICE, "GetDCHotNodeListByPlugin:Batch Number is invalid." .. BatchNo);
        return nil;
    end

    LogText(LL_NOTICE, "GetDCHotNodeListByPlugin:Batch Number." .. BatchNo);
    return nil;
end


function GetFirstNodeList()
 LogText(LL_NOTICE, "Get First NodeList.");

    local FirstBatch = {};
    local MasterNid = "0";
    local FirstNid = "0";
    local Index = 0;

    for _, Node in pairs(g_UpdState.g_MasterSlave) do
        if (Node.IsMaster == "1" ) then
            if(Node.Nid == "0") then
                MasterNid = "0";
                FirstNid = "2";
            elseif(Node.Nid == "1") then
                MasterNid = "1";
                FirstNid = "2";
            else
                MasterNid = "2";
                FirstNid = "0";
            end
            Index = GetNodeIndex(FirstNid);
            if (nil == Index) then
                --继续查找与主不再同一个平面的节点作为第一批
                for _, Node in pairs(g_UpdState.g_ClusterInfo) do
                    if ((BatchNo == 1) and ((Node.Nid%2) ~= (GetMasterNid()%2))) then ---
                        LogText(LL_NOTICE, "Insert FirstBatch node:" .. Node.Nid);
                        table.insert(FirstBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 1});
                        g_UpdState.NodeList[ip2int(Node.ip)].batchno = 1;
                    end
                end
                --不存在与主不在一个平面的节点，把所有的节点作为第一批
                if(GetSizeOfTable(FirstBatch) == 0) then
                    for _, Node in pairs(g_UpdState.g_ClusterInfo) do
                        LogText(LL_NOTICE, "Insert FirstBatch node:" .. Node.Nid);
                        table.insert(FirstBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 1});
                       g_UpdState.NodeList[ip2int(Node.ip)].batchno = 1;
                    end
                end
            else
                CurNode = g_UpdState.g_MasterSlave[Index];
                LogText(LL_NOTICE, "Insert FirstBatch node:" .. CurNode.Nid);
                table.insert(FirstBatch,{ip = CurNode.ip, type = CurNode.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 1});
                g_UpdState.NodeList[ip2int(CurNode.ip)].batchno = 1;
            end
        end
    end

    SaveState(g_UpdState);
    return FirstBatch;
end

function GetSecondNodeList()
    LogText(LL_NOTICE, "Get Second Nodelist.");
    local SecondBatch = {};

    for _, Node in pairs(g_UpdState.g_ClusterInfo) do
        if(((Node.Nid%2) ~= (GetMasterNid()%2)) and IsNotInFirstBatch(Node.ip)) then
            LogText(LL_NOTICE, "Insert SecondBatch node:" ..Node.Nid);
            table.insert(SecondBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 2});
            g_UpdState.NodeList[ip2int(Node.ip)].batchno = 2;
        end
    end

    if(GetSizeOfTable(SecondBatch) == 0) then
        for _, Node in pairs(g_UpdState.g_ClusterInfo) do
            if(IsNotInFirstBatch(Node.ip)) then
                LogText(LL_NOTICE, "Insert SecondBatch node:" .. Node.Nid);
                table.insert(SecondBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 2});
                g_UpdState.NodeList[ip2int(Node.ip)].batchno = 2;
            end
        end
    end

    SaveState(g_UpdState);
    return SecondBatch;
end

function GetThirdNodeList()
    LogText(LL_NOTICE, "Get Third Nodelist.");
    local ThirdBatch = {};

    for _, Node in pairs(g_UpdState.g_ClusterInfo) do
        if(IsInLastBatch(Node.ip)) then
            LogText(LL_NOTICE, "Insert ThirdBatch node:" .. Node.Nid);
            table.insert(ThirdBatch,{ip = Node.ip, type = Node.tp, state = ATOM_GROUP_EXEC_RET_INIT, batchno = 3});
            g_UpdState.NodeList[ip2int(Node.ip)].batchno = 3;
        end
    end

    SaveState(g_UpdState);
    return ThirdBatch;
end

function GetMCHotNodeListByPlugin(BatchNo)
    LogText(LL_NOTICE, "This is multi controller.");

    if (BatchNo == 1)then
        --第一批次执行之前，首先将各个节点的batchno设置成0，避免升级时候的批次信息影响回退批次的计算
        for _, Node in pairs(g_UpdState.NodeList) do
            g_UpdState.NodeList[ip2int(Node.ip)].batchno = 0;
        end
        return GetFirstNodeList();

    elseif (BatchNo == 2)then
        return GetSecondNodeList();

    elseif (BatchNo == 3)then
        return GetThirdNodeList();
    else
        LogText(LL_NOTICE, "GetMCHotNodeListByPlugin:Batch Number is invalid." );
        return nil;
    end

end
--------------------------------------------------------------------------------
-- Func Name       :GetHotNodeListByPlugin
-- Description     :热补丁回退时，获取回退批次的节点信息
-- Caution         :当前主要用于热补丁升级失败后执行大回退
-- Input           :BatchNo 回退的批次
-- Return          :回退批次对应的节点列表
--------------------------------------------------------------------------------
function GetHotNodeListByPlugin(BatchNo)
    LogText(LL_NOTICE, "GetHotNodeListByPlugin, BatchNo: " .. BatchNo);

    -- 如果是单控，只有第一批有节点
    if(GetSizeOfTable(g_UpdState.NodeList) == 1 and #g_UpdState.g_MasterSlave == 1) then
        return GetSCHotNodeListByPlugin(BatchNo);
    --双控
    elseif(GetSizeOfTable(g_UpdState.NodeList) == 2) then
        return GetDCHotNodeListByPlugin(BatchNo);
    --多控
    else
        return GetMCHotNodeListByPlugin(BatchNo);
    end
end

--------------------------------------------------------------------------------
-- Func Name       :BatchRunCurAtomGroup
-- Description     :热补丁回退时，获取回退批次的节点信息
-- Caution         :当前主要用于热补丁升级失败后执行大回退
-- Input           :BatchNo 回退的批次
-- Return          :回退批次对应的节点列表
--------------------------------------------------------------------------------
function BatchRunCurAtomGroup()
    LogText(LL_NOTICE, "Exec BatchRunCurAtomGroup.");
    --所有批次都执行完当前的原子组后，再获取下一个原子组
    local BatchNo = g_UpdState.BatchNo;
    local NodeList = g_UpdState.CurNodeList;
    local AGIndex = g_UpdState.AtomGroupIndex;
    local NextNodeList = nil;
    local RbkBatchNo = g_UpdState.RbkBatchNo;

    --检查当前批次节点是否都执行完成，当前批次的节点执行完成
    local isFinish = true;
    for _, CurNode in pairs(NodeList) do
        if (CurNode.state == ATOM_GROUP_EXEC_RET_INIT) then --未开始执行
            LogText(LL_NOTICE, "Some Node has not exec finish.");
            isFinish = false;
        end
    end

    --AslPatchUpgrade升级中第一个原子组失败，直接退出
      if( 1 == g_UpdState.AtomGroupIndex and 0 == g_UpdState.FlowUpdState and 0 ~= #g_UpdState.FailNodeList and
        "AslPatchUpgrade" == g_UpdState.Flow) then
            LogText(LL_NOTICE, "The AslPatchUpgrade failed when execute CopyAslPatch AG and terminate the AslPatchUpgrade flow ");
            UpdateNodeState(END_BATCH, g_UpdState.CurNodeList, "Manager");
            UpdateNodeState(TERM_FLOW, g_UpdState.NodeList, "Manager");
            g_UpdState.FlowState = FLOW_STATE_FINISH;
            SaveState(g_UpdState);
            return FLOW_DISPATCH_STATE_FINISH;
      end


    --热补丁回退、删除过程中发生异常重启，则从当前批次继续回退、删除
    for _, node in pairs(g_UpdState.CurNodeList) do
        if(5 == g_UpdState.NodeList[ip2int(node.ip)].rt) then
            ABNORMAL_REBOOT_WHEN_HOTPATCH_ROLLBACK_OR_DELETE = 1;
            return FLOW_DISPATCH_STATE_NORMAL;
        end
    end

    --补丁安装失败回退，回退时失败则终止flow;补丁回退删除失败，某个批次存在失败节点时，终止回退/删除
    if((MANAGER_FLOW_STATE_UPG_RBK == g_UpdState.FlowUpdState) and 0~= RbkBatchNo and IsRbkFail() and (g_UpdState.g_ClusInfoChange == 0))
        or (g_UpdState.Flow ~= "HotPatchUpgrade" and g_UpdState.Flow ~= "AslPatchUpgrade" and #g_UpdState.FailNodeList ~= 0 ) then
        LogText(LL_NOTICE, "Some nodes rollback or delete fail and terminate the flow.");
        ---首先更新失败节点的状态，然后将未执行的节点设置成取消状态
        UpdateNodeState(END_BATCH, g_UpdState.CurNodeList, "Manager");
        UpdateNodeState(TERM_FLOW, g_UpdState.NodeList, "Manager");
        g_UpdState.FlowState = FLOW_STATE_FINISH;
        SaveState(g_UpdState);
        return FLOW_DISPATCH_STATE_FINISH;
    end

    --当某个批次执行完成一个原子组时，满足此分支，此时需要直接返回，开始下一批次执行当前的原子组
    if (0 == g_UpdState.FlowUpdState)then
        LogText(LL_NOTICE, "Begin upd next batch :" .. BatchNo+1);
        NextNodeList = GetHotNodeListByPlugin(BatchNo+1);
    end
    if (1 == g_UpdState.FlowUpdState)then
        LogText(LL_NOTICE, "Begin rbk next batch :" .. RbkBatchNo+1);
        NextNodeList = GetHotNodeListByPlugin(RbkBatchNo+1);
    end
    if(0 ~= AGIndex) and (isFinish) and (nil ~= NextNodeList) and (0 ~= #NextNodeList)then
        if (g_UpdState.FlowUpdState == 0)then
            LogText(LL_NOTICE, "Current Batch:" .. BatchNo .. " finish the cur atom group." );
        end
        if (g_UpdState.FlowUpdState == 1)then
            LogText(LL_NOTICE, "Current RbkBatch:" .. RbkBatchNo .. " finish the cur atom group." );
        end
        UpdateNodeState(END_BATCH, g_UpdState.CurNodeList, "Manager");
        SaveState(g_UpdState);
        --当前失败节点为0或者，安装失败回退过程中，当前批次不存在失败节点时，继续执行下一个批次
        if((0 == g_UpdState.FlowUpdState) and ((("HotPatchUpgrade" == g_UpdState.Flow) and (0 ~= #g_UpdState.FailNodeList))
            or (("AslPatchUpgrade" == g_UpdState.Flow) and (0 ~= #g_UpdState.FailNodeList)) or (1 == g_UpdState.g_ClusInfoChange ))) then
            LogText(LL_NOTICE, "Some Nodes Upgrade Fail and Reset the g_UpdState.FlowUpdState and g_UpdState.AtomGroupIndex");
            --补丁升级的过程中节点异常重启后，为了避免节点恢复之前执行回退导致回退失败，此处睡眠3分钟
            g_UpdState.FlowUpdState= MANAGER_FLOW_STATE_UPG_RBK;
            g_UpdState.FailRbkAGIndex = g_UpdState.AtomGroupIndex;
            g_UpdState.AtomGroupIndex= 0;
            AGIndex= g_UpdState.AtomGroupIndex;
        else
            LogText(LL_NOTICE, "Exec the next batch.");
            return FLOW_DISPATCH_STATE_NORMAL;
        end
    end
    ---安装补丁最后一个批次也需要判断是否有失败节点，当有失败时，需要更新g_UpdState.FlowUpdState，并设置AtomGroupIndex为从第一个原子组开始
    if(nil == NextNodeList) or (0 == #NextNodeList) or (0 == AGIndex) then --第一个批次开始或者最后一个批次结束，都需要将AG加1，并且从第一个批次开始执行新的原子组
        if (0 == AGIndex) then
            LogText(LL_NOTICE, "Start run first atom group.");
        else
            LogText(LL_NOTICE, "All nodes finish the cur atom group.");
        end
        UpdateNodeState(END_BATCH, g_UpdState.CurNodeList, "Manager");
        if (g_UpdState.FlowUpdState == 0) and (((g_UpdState.Flow == "HotPatchUpgrade") and (#g_UpdState.FailNodeList ~= 0)) or
        ((g_UpdState.Flow == "AslPatchUpgrade") and (#g_UpdState.FailNodeList ~= 0)) or (g_UpdState.g_ClusInfoChange == 1)) then
            LogText(LL_NOTICE, "Last Batch Nodes Upgrade Fail and Reset the g_UpdState.FlowUpdState and g_UpdState.AtomGroupIndex");
            g_UpdState.FlowUpdState = MANAGER_FLOW_STATE_UPG_RBK;
            g_UpdState.AtomGroupIndex = 0;
            g_UpdState.FailRbkAGIndex = g_UpdState.AtomGroupIndex;
            AGIndex = g_UpdState.AtomGroupIndex + 1;
            SaveState(g_UpdState);
        else
            AGIndex = g_UpdState.AtomGroupIndex + 1;
            SaveState(g_UpdState);
        end
        if ((g_UpdState.FlowUpdState == 0 and AGIndex > #g_UpdateFlowsTbl[g_UpdState.Flow].upd)
            or (g_UpdState.FlowUpdState == 1 and AGIndex > #g_UpdateFlowsTbl[g_UpdState.Flow].rbk)) then
            g_UpdState.FlowState = FLOW_STATE_FINISH;
            UpdateNodeState(TERM_FLOW, g_UpdState.NodeList, "Manager");
            SaveState(g_UpdState);
            return FLOW_DISPATCH_STATE_FINISH;
        end
    end
    --从第一个批次开始执行AG
    BatchNo = 0;
    RbkBatchNo = 0;
    g_UpdState.BatchNo = BatchNo;
    g_UpdState.RbkBatchNo = RbkBatchNo;
    g_UpdState.AtomGroupIndex = AGIndex;
    SaveState(g_UpdState);

    return FLOW_DISPATCH_STATE_NORMAL;
end

--------------------------------------------------------------------------------
-- Func Name       :ReStartRbkFlow
-- Description     :热补丁升级过程发生异常主备倒换后，设置开始回退的条件
--------------------------------------------------------------------------------
function ReStartRbkFlow()
---进行初始化，使流程可以开始执行回退-----------
    LogText(LL_NOTICE, "Exec ReStartRbkFlow.");
    if("HotPatchUpgrade" ~= g_UpdState.Flow and "AslPatchUpgrade" ~= g_UpdState.Flow)then
        LogText(LL_NOTICE, "FlowName is" .. g_UpdState.Flow);
        return 0;
    end
    if(1 == g_UpdState.FlowUpdState)then
        LogText(LL_NOTICE, "The flowupdstate is rbk now.");
        return 0;
    end
    if(1 == g_UpdState.g_ClusInfoChange)then
        LogText(LL_NOTICE, "The g_ClusInfo is Changed before.");
        return 0;
    end
    g_UpdState.g_ClusInfoChange = 1;
    SaveState(g_UpdState);

    LogText(LL_NOTICE, "Set g_UpdState.g_ClusInfoChange:" .. g_UpdState.g_ClusInfoChange);
    return 1;
end

--------------------------------------------------------------------------------
-- Func Name       :RunHotpatchUpd
-- Description     :热补丁升级流程功能函数
--------------------------------------------------------------------------------
function RunHotpatchUpd()
    LogText(LL_NOTICE, "Exec RunHotpatchUpd.");
    if(FLOW_DISPATCH_STATE_FINISH == BatchRunCurAtomGroup())then
        LogText(LL_NOTICE, "BatchRunCurAtomGroup Finished.");
        g_UpdState.FlowState = FLOW_STATE_FINISH;
        SaveState(g_UpdState);
        PostHandleAfterUpdFinish(g_UpdState.Flow);
        return FLOW_DISPATCH_STATE_FINISH;
    end
    if(g_UpdState.FlowUpdState == 0)then
        local BatchNo = g_UpdState.BatchNo + 1;---第一批次执行
        g_UpdState.BatchNo = BatchNo;
        LogText(LL_NOTICE, "Begin exec upgrade from g_UpdState.BatchNo:" .. g_UpdState.BatchNo);
        NodeList = GetHotNodeListByPlugin(BatchNo);
        ---当集群主重启后,主备发生切换,再次获取批次信息时,可能会导致第一批次节点为nil, 此时走回退流程
        if((nil == NodeList) and (1 == BatchNo))then
            g_UpdState.FlowUpdState = 1;
            g_UpdState.RbkBatchNo = 0; --开始回退流程
            g_UpdState.AtomGroupIndex = 1;--开始回退流程，改变AG的值
            for _, Node in pairs(g_UpdState.CurNodeList) do
                Node.state = ATOM_GROUP_EXEC_RET_INIT;
            end
            SaveState(g_UpdState);
        end
    end
    ---由于升级的分支可能改变成回退，所以此处写两个if语句
    if(g_UpdState.FlowUpdState == 1)then
        local RbkBatchNo = g_UpdState.RbkBatchNo + 1;---第一批次执行
        g_UpdState.RbkBatchNo = RbkBatchNo;
        LogText(LL_NOTICE, "Begin exec rollback from g_UpdState.RbkBatchNo:" .. g_UpdState.RbkBatchNo);
        NodeList = GetHotNodeListByPlugin(RbkBatchNo);
    end

    g_UpdState.CurNodeList = NodeList;
    LogText(LL_NOTICE, "----" .. g_UpdState.RbkBatchNo .. g_UpdState.AtomGroupIndex);
    UpdateNodeState(START_BATCH, NodeList, "Manager");
    SaveState(g_UpdState);

    local AGIndex = g_UpdState.AtomGroupIndex;

    if(g_UpdState.FlowUpdState == 0)then
        local AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[AGIndex];--每一批次的节点执行当前的原子组
        for _, Node in pairs(g_UpdState.CurNodeList) do
            if (Node.state == ATOM_GROUP_EXEC_RET_INIT ) or (Node.state == ATOM_GROUP_EXEC_RET_SUCC) then
                Node.state = ATOM_GROUP_EXEC_RET_INIT; --重置节点执行状态
                NodeExecAtomGroup(Node, AtomGroup, Table2String(g_UpdState.ParamList));
            end
        end
    else
        local AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].rbk[AGIndex];--每一批次的节点执行当前的原子组
        for _, Node in pairs(g_UpdState.CurNodeList) do
            if (Node.state == ATOM_GROUP_EXEC_RET_INIT ) or (Node.state == ATOM_GROUP_EXEC_RET_SUCC_RBK) then
                Node.state = ATOM_GROUP_EXEC_RET_INIT; --重置节点执行状态
                NodeExecAtomGroup(Node, AtomGroup, Table2String(g_UpdState.ParamList));
            end
        end
    end
    return FLOW_DISPATCH_STATE_NORMAL;
end

--------------------------------------------------------------------------------
-- Func Name       :RunHotpatchRbk
-- Description     :热补丁回退删除流程功能函数
--------------------------------------------------------------------------------
function RunHotpatchRbk()
    LogText(LL_NOTICE, "Exec RunHotpatchRbk.");
    if(FLOW_DISPATCH_STATE_FINISH == BatchRunCurAtomGroup())then
        LogText(LL_NOTICE, "BatchRunCurAtomGroup Finished.");
        g_UpdState.FlowState = FLOW_STATE_FINISH;
        SaveState(g_UpdState);
        PostHandleAfterUpdFinish(g_UpdState.Flow);
        return FLOW_DISPATCH_STATE_FINISH;
    end

    local BatchNo = g_UpdState.BatchNo + 1;---第一批次执行

    --如果在热补丁回退、删除发生异常重启，则让其从当前批次执行操作
    if(1 == ABNORMAL_REBOOT_WHEN_HOTPATCH_ROLLBACK_OR_DELETE)then
        BatchNo = BatchNo - 1;
        ABNORMAL_REBOOT_WHEN_HOTPATCH_ROLLBACK_OR_DELETE = 0;
        LogText(LL_NOTICE, "Current batch is: " .. BatchNo);
    end

    g_UpdState.BatchNo = BatchNo;
    LogText(LL_NOTICE, "RunHotpatchRbk begin exec upgrade from g_UpdState.BatchNo:" .. g_UpdState.BatchNo)
    NodeList = GetHotNodeListByPlugin(BatchNo);
    g_UpdState.CurNodeList = NodeList;
    UpdateNodeState(START_BATCH, NodeList, "Manager");
    SaveState(g_UpdState);

    local AGIndex = g_UpdState.AtomGroupIndex;
    local AtomGroup = g_UpdateFlowsTbl[g_UpdState.Flow].upd[AGIndex];--每一批次的节点执行当前的原子组
    --若为回退流程，则回退成功的节点继续执行流程
    if(g_UpdateFlowsTbl[g_UpdState.Flow].isRbk)then
        for _, Node in pairs(g_UpdState.CurNodeList) do
            if (Node.state == ATOM_GROUP_EXEC_RET_INIT ) or (Node.state == ATOM_GROUP_EXEC_RET_SUCC_RBK) then
                Node.state = ATOM_GROUP_EXEC_RET_INIT; --重置节点执行状态
                NodeExecAtomGroup(Node, AtomGroup, Table2String(g_UpdState.ParamList));
            end
        end
        return FLOW_DISPATCH_STATE_NORMAL;
    else
        for _, Node in pairs(g_UpdState.CurNodeList) do
            if (Node.state == ATOM_GROUP_EXEC_RET_INIT ) or (Node.state == ATOM_GROUP_EXEC_RET_SUCC) then
                Node.state = ATOM_GROUP_EXEC_RET_INIT; --重置节点执行状态
                NodeExecAtomGroup(Node, AtomGroup, Table2String(g_UpdState.ParamList));
            end
        end
        return FLOW_DISPATCH_STATE_NORMAL;
    end
end

function SetSwapMasterFlag(swapMasterFlag)
    LogText(LL_NOTICE, "NotifySysCtrlPrepareForOnline succeed, Set g_SwapMasterFalg 0");
	g_UpdState.g_SwapMasterFalg = swapMasterFlag;
	return 0;
end
--升级流程表
g_UpdateFlowsTbl =
{
    --上电同步flow原子组配置
    SyncUpgrade =
    {
        info = "Manager:SyncSysAndFirmware=30,SyncPcieSwitch=50,NotifySysReboot=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 95,
        timeout = 102,
        upd =
        {
            {AtomGroupName = "SyncSysAndFirmware", ExcuteNodeType = "ALL", timeout = 100},
            {AtomGroupName = "SyncPcieSwitch", ExcuteNodeType = "Manager" , timeout = 30},
            {AtomGroupName = "NotifySysReboot", ExcuteNodeType = "Manager" , timeout = 20},
        },
    },
    --插入框flow原子组配置
    InsertFRM =
    {
        info = "Manager:SyncFrame=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 33,
        timeout = 32,
        upd =
        {
            {AtomGroupName = "SyncFrame", timeout = 31},
        },
    },
    --插入BBU原子组配置
    InsertBBU =
    {
        info = "Manager:SyncBBU=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 47,
        timeout = 46,
        upd =
        {
            {AtomGroupName = "SyncBBU", timeout = 45},
        },
    },
    --BBU后台升级原子组配置
    InsertBBUBackGround =
    {
        info = "Manager:SyncBBU=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 47,
        timeout = 46,
        upd =
        {
            {AtomGroupName = "SyncBBU", timeout = 45},
        },
    },
    --插入SAS原子组配置
    InsertSAS =
    {
        info = "Manager:SyncSAS=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 12,
        timeout = 10,
        upd =
        {
            {AtomGroupName = "SyncSAS", timeout = 9},
        },
    },
    --插入FCoE原子组配置
    InsertFCoE =
    {
        info = "Manager:SyncFCoE=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 12,
        timeout = 10,
        upd =
        {
            {AtomGroupName = "SyncFCoE", timeout = 9},
        },
    },
    --插入TOE原子组配置
    InsertTOE =
    {
        info = "Manager:SyncTOE=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 12,
        timeout = 10,
        upd =
        {
            {AtomGroupName = "SyncTOE",timeout = 9},
        },
    },
    --插入PCIE接口卡原子组配置
    InsertPCIE =
    {
        info = "Manager:SyncPCIE=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 12,
        timeout = 10,
        upd =
        {
            {AtomGroupName = "SyncPCIE", timeout = 9},
        },
    },
    --插入12GSAS接口卡
    Insert12GSAS =
    {
        info = "Manager:Sync12GSAS=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 12,
        timeout = 10,
        upd =
        {
            {AtomGroupName = "Sync12GSAS", timeout = 9},
        },
    },
    --插入12GSAS接口卡
    InsertSHARECARD12GSAS =
    {
        info = "Manager:Sync12GSAS=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 12,
        timeout = 10,
        upd =
        {
            {AtomGroupName = "Sync12GSAS", timeout = 9},
        },
    },
    --插入16GFC接口卡
    Insert16GFC =
    {
        info = "Manager:Sync16GFC=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 12,
        timeout = 10,
        upd =
        {
            {AtomGroupName = "Sync16GFC", timeout = 9},
        },
    },
    --插入MGMT原子组配置
    InsertMGMT =
    {
        info = "Manager:SyncMGMT=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 33,
        timeout = 32,
        upd =
        {
            {AtomGroupName = "SyncMGMT", timeout = 31},
        },
    },
    --MGMT后台升级原子组配置
    InsertMGMTBackGround =
    {
        info = "Manager:SyncMGMT=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 33,
        timeout = 32,
        upd =
        {
            {AtomGroupName = "SyncMGMT", timeout = 31},
        },
    },
    --插入MGMT接口卡原子组配置
    InsertIXGBE =
    {
        info = "Manager:SyncIXGBE=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 12,
        timeout = 10,
        upd =
        {
            {AtomGroupName = "SyncIXGBE", timeout = 9},
        },
    },
    --插入56G IB接口卡原子组配置
    Insert56GIB =
    {
        info = "Manager:Sync56GIB=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 12,
        timeout = 10,
        upd =
        {
            {AtomGroupName = "Sync56GIB", timeout = 9},
        },
    },
    --插入ACC接口卡原子组配置
    InsertACCFPGA =
    {
        info = "Manager:SyncACC=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 13,
        timeout = 12,
        upd =
        {
            {AtomGroupName = "SyncACC", timeout = 11},
        },
    },
    --插入10GBaseT接口卡原子组配置
    Insert10GBaseT =
    {
        info = "Manager:Sync10GBaseT=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 12,
        timeout = 10,
        upd =
        {
            {AtomGroupName = "Sync10GBaseT", timeout = 9},
        },
    },
    --插入SmartIO接口卡原子组配置
    InsertSmartIO =
    {
        info = "Manager:SyncSmartIO=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 12,
        timeout = 10,
        upd =
        {
            {AtomGroupName = "SyncSmartIO", timeout = 9},
        },
    },
    --更换IP Scale-out组网的SmartIO卡
    InsertSmartIOInOfflineStatus =
    {
        info = "Manager:SyncSmartIOInOfflineStatus=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 12,
        timeout = 10,
        upd =
        {
            {AtomGroupName = "SyncSmartIOInOfflineStatus", timeout = 9},
        },
    },
    --插入IOCHD接口卡原子组配置
    InsertIOCHD =
    {
        info = "Manager:SyncIOCHD=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 13,
        timeout = 12,
        upd =
        {
            {AtomGroupName = "SyncIOCHD", timeout = 11},
        },
    },
    --插入IOC10GE接口卡原子组配置
    InsertIOC10GE =
    {
        info = "Manager:SyncIOC10GE=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 13,
        timeout = 12,
        upd =
        {
            {AtomGroupName = "SyncIOC10GE", timeout = 11},
        },
    },
    --插入IOC8GFC接口卡原子组配置
    InsertIOC8G =
    {
        info = "Manager:SyncIOC8G=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 13,
        timeout = 12,
        upd =
        {
            {AtomGroupName = "SyncIOC8G", timeout = 11},
        },
    },
    --插入IOC16GFC接口卡原子组配置
    InsertIOC16G =
    {
        info = "Manager:SyncIOC16G=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 13,
        timeout = 12,
        upd =
        {
            {AtomGroupName = "SyncIOC16G", timeout = 11},
        },
    },
    --插入PCIE交换机原子组配置
    InsertPCIESW =
    {
        info = "Manager:InsertSyncPcieSwitch=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 12,
        timeout = 32,
        upd =
        {
            {AtomGroupName = "InsertSyncPcieSwitch", timeout = 30},
        },
    },
    -- 同步SAS大卡
    InsertSHARECARD =
    {
        info = "Manager:InsertSHARECARD=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 43,
        timeout = 42,
        upd =
        {
            {AtomGroupName = "SyncSHARECARD", timeout = 41},
        },
    },
    ------分发包流程:
    DownloadPkg =
    {
        info = "Manager:RepaireSysdisk=10,CopyPackage=30,UnzipUpgradePackage=70,VerifyUpgradePackage=100,CreateUpdStateFile=5",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 39,
        timeout = 40,
        upd =
        {
            {AtomGroupName = "RepaireSysdisk", timeout = 2},
            {AtomGroupName = "CopyPackage", timeout = 26},
            {AtomGroupName = "UnzipUpgradePackage", timeout = 8},
            {AtomGroupName = "VerifyUpgradePackage", timeout = 5},
        },
    },
    OnlineUpgrade =
    {
        info = "Manager:UploadPreupd=5,UpgradePcieSw=10,UploadPackageWhenOnlineUpgrade=20,AgentPrepareForOnlineUpgrade=40,ManagerPrepareForOnlineUpgrade=50,UpgradeSystemWhenOnlineUpgrade=60,RebootWhenOnlineUpgrade=80,SysCompatibleAfterReboot=90,SysMirrorRecoveryCheck=20,UpgradePostupd=100,UploadPackageWhenOnlineRollback=20,PrepareForUpdateWhenOnlineRollback=40,RollbackSystemWhenOnlineUpgrade=80,RebootWhenOnlineRollback=90,SysCompatibleAfterRollbackReboot=95,UpgradePcieSwRollback=100,AbnormalSwapMaster=50,RebootFailNode=90,NotifySysStartUpgrade=5,NotifySysUpgradeFinish=100,CreateUpdStateFile=5",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = true,
        isInBatches = true,
        totalTime = 292,
        timeout = 291,

        preupd =
        {
            {AtomGroupName = "UploadPreupd", timeout = 90},
        },

        upd =
        {
            {AtomGroupName = "UpgradePcieSw", ExcuteNodeType = "Manager", timeout = 35, rbk = "UpgradePcieSwRollback"},
            {AtomGroupName = "UploadPackageWhenOnlineUpgrade", timeout = 30, rbk = "UpgradePcieSwRollback"},
            {AtomGroupName = "AgentPrepareForOnlineUpgrade", timeout = 245, rbk = "RebootWhenOnlineRollback"},
            {AtomGroupName = "ManagerPrepareForOnlineUpgrade", timeout = 245, rbk = "RebootWhenOnlineRollback"},
            {AtomGroupName = "UpgradeSystemWhenOnlineUpgrade", timeout = 85, rbk = "UploadPackageWhenOnlineRollback"},
            {AtomGroupName = "RebootWhenOnlineUpgrade", ExcuteNodeType = "ALL", timeout = 60, rbk = "UploadPackageWhenOnlineRollback"},
            {AtomGroupName = "SysCompatibleAfterReboot", timeout = 90, rbk = "UploadPackageWhenOnlineRollback"},
            {AtomGroupName = "SysMirrorRecoveryCheck", ExcuteNodeType = "Manager", timeout = 130, rbk = "UploadPackageWhenOnlineRollback"},
        },
        rbk =
        {
            {AtomGroupName = "UploadPackageWhenOnlineRollback", timeout = 15},
            {AtomGroupName = "PrepareForUpdateWhenOnlineRollback", timeout = 290},
            {AtomGroupName = "RollbackSystemWhenOnlineUpgrade", timeout = 83},
            {AtomGroupName = "RebootWhenOnlineRollback",ExcuteNodeType = "ALL", timeout = 40},
            {AtomGroupName = "SysCompatibleAfterRollbackReboot", timeout = 130},
            {AtomGroupName = "UpgradePcieSwRollback", ExcuteNodeType = "Manager", timeout = 30},
        },

        postupd =
        {
            {AtomGroupName = "UpgradePostupd", timeout = 90},
        },
    },

    --离线升级及回退
    OfflineUpgrade =
    {
        info = "Manager:UploadPreupd=5,UploadPackageWhenOfflineUpgrade=10,PrepareForUpdateWhenOfflineUpgrade=30,UpgradePcieSw=40,UpgradeSystemWhenOfflineUpgrade=80,RebootWhenOfflineUpgrade=90,OfflineSysCompatibleAfterReboot=90,UpgradePostupd=100,UploadFirmwareWhenOfflineUpgradeRollback=30,PrepareForUpdateWhenOfflineUpgradeRbk=40,RollbackSystemWhenOfflineUpgrade=80,UpgradePcieSwRollback=90,RebootWhenOfflineRollback=95,SysCompatibleAfterRollbackReboot=100,RebootFailNode=90,NotifySysStartUpgrade=5,NotifySysUpgradeFinish=100,CreateUpdStateFile=5",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = true,
        isInBatches = false,
        totalTime = 100,
        timeout = 90,

        preupd =
        {
            {AtomGroupName = "UploadPreupd", timeout = 90},
        },

        upd =
        {
            {AtomGroupName = "UploadPackageWhenOfflineUpgrade", timeout = 30, rbk = "", IsNeedReboot = false},
            {AtomGroupName = "PrepareForUpdateWhenOfflineUpgrade", ExcuteNodeType = "Manager", timeout = 45, rbk = "RebootWhenOfflineRollback", IsNeedReboot = true},
            {AtomGroupName = "UpgradePcieSw", ExcuteNodeType = "Manager", timeout = 35, rbk = "UpgradePcieSwRollback", IsNeedReboot = true},
            {AtomGroupName = "UpgradeSystemWhenOfflineUpgrade", timeout = 85, rbk = "UploadFirmwareWhenOfflineUpgradeRollback", IsNeedReboot = true},
            {AtomGroupName = "RebootWhenOfflineUpgrade", ExcuteNodeType = "ALL", timeout = 60, rbk = "UploadFirmwareWhenOfflineUpgradeRollback", IsNeedReboot = true},
            {AtomGroupName = "OfflineSysCompatibleAfterReboot", timeout = 90, rbk = "UploadFirmwareWhenOfflineUpgradeRollback", IsNeedReboot = true},
        },
        rbk =
        {
            {AtomGroupName = "UploadFirmwareWhenOfflineUpgradeRollback", timeout = 15},
            --只有当节点重启后的兼容性校验失败大回退时，才进行离线准备，其他情况不需要执行准备操作
            {AtomGroupName = "PrepareForUpdateWhenOfflineUpgradeRbk", ExcuteNodeType = "Manager", timeout = 30},
            {AtomGroupName = "RollbackSystemWhenOfflineUpgrade", timeout = 83},
            {AtomGroupName = "RebootWhenOfflineRollback", ExcuteNodeType = "ALL", timeout = 40},
            {AtomGroupName = "SysCompatibleAfterRollbackReboot", timeout = 90},
            {AtomGroupName = "UpgradePcieSwRollback", ExcuteNodeType = "Manager", timeout = 30},
        },

        postupd =
        {
            {AtomGroupName = "UpgradePostupd", timeout = 90},
        },
    },

    --热补丁升级-----
    HotPatchUpgrade =
    {
        info = "Manager:InstallHotpatch=70,ConfirmHotPatch=100,RollbackHotPatchWhenFail=100,NotifySysStartUpgrade=5,NotifySysUpgradeFinish=100,CreateUpdStateFile=5",
        func = RunHotpatchUpd,
        isRbk = false,
        isInBatches = true,
        totalTime = 12,
        timeout = 25,
        upd =
        {
            {AtomGroupName = "InstallHotpatch", timeout = 20, rbk = ""},
            {AtomGroupName = "ConfirmHotPatch", timeout = 15, rbk = ""},
        },
        rbk =
        {
            {AtomGroupName = "RollbackHotPatchWhenFail", timeout = 3},
        },
    },
    --热补丁回退
    HotPatchRollback =
    {
        info = "Manager:RollbackHotpatchCheck=40,RollbackHotpatch=100,NotifySysStartUpgrade=5,NotifySysUpgradeFinish=100,CreateUpdStateFile=5",
        func = RunHotpatchRbk,
        isRbk = true,
        isInBatches = true,
        totalTime = 36,
        timeout = 35,
        upd =
        {
            {AtomGroupName = "RollbackHotpatchCheck", timeout = 4},
            {AtomGroupName = "RollbackHotpatch", timeout = 32},
        },
    },
    --热补丁删除
    HotPatchDelete =
    {
        info = "Manager:DeleteHotpatchCheck=40,DeleteHotpatch=100,NotifySysStartUpgrade=5,NotifySysUpgradeFinish=100,CreateUpdStateFile=5",
        func = RunHotpatchRbk,
        isRbk = false,
        isInBatches = true,
        totalTime = 36,
        timeout = 36,
        upd =
        {
            {AtomGroupName = "DeleteHotpatchCheck", timeout = 4, rbk = ""},
            {AtomGroupName = "DeleteHotpatch",timeout = 32, rbk = ""},
        },
    },
    --异构包升级-----
    AslPatchUpgrade=
    {
        info= "Manager:CopyAslPatch=70,InstallAslPatch=80,ResetLinkToAslPatch=100,RollbackAslPatchWhenFail=100,NotifySysStartUpgrade=5,NotifySysUpgradeFinish=100,CreateUpdStateFile=5",
        func= RunHotpatchUpd,
        isRbk= false,
        isInBatches= true,
        totalTime= 12,
        timeout= 25,
        upd=
        {
            {AtomGroupName= "CopyAslPatch", timeout= 20, rbk= ""},
            {AtomGroupName= "InstallAslPatch", timeout= 15, rbk= "RollbackAslPatchWhenFail"},
            {AtomGroupName= "ResetLinkToAslPatch", timeout= 15, rbk= "RollbackAslPatchWhenFail"},
        },
        rbk=
        {
            {AtomGroupName= "RollbackAslPatchWhenFail", timeout= 15},
        },
    },
    --特殊补丁升级，用于调试或生产环节
    SpecialPatchUpgrade =
    {
        info = "Manager:InstallSpecialPatch=100",
        func = RunFlowFunc,
        isRbk = false,
        isFlowRbk = false,
        isInBatches = false,
        totalTime = 8,
        timeout = 7,
        upd =
        {
            {AtomGroupName = "InstallSpecialPatch", timeout = 6, rbk = ""},
        },
    },
}
