#!/bin/bash

#------------------------------------------------------------------------
# August, 2015, Balaji Krishnamoorthy
# Copyright (c) 2015-2017, 2019 by Cisco Systems, Inc.
# All Rights Reserved
#------------------------------------------------------------------------



echo $PATH | grep -q -o "/opt/cisco/calvados/bin"
if [ $? != 0 ]; then
    export PATH=/opt/cisco/calvados/bin:$PATH
fi

echo $PATH | grep -q -o "/opt/cisco/calvados/sbin"
if [ $? != 0 ]; then
    export PATH=/opt/cisco/calvados/sbin:$PATH
fi

MYADDR=$(ifconfig eth-vf1.3073 | sed -n -e '/inet addr/s/^.*inet addr:\([1-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\).*$/\1/p')

CONFDADDR=$(ds_test_client -l confd6 | sed -n -e '/ha_role=ACTIVE/s/^.*, *ip=\([1-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\).*$/\1/p')

print_usage() {
    echo ""
    echo "  USAGE : $0 fia|s1|s2|s3 rx|tx linkhandle"
    echo "     Dumps debug information for devices in the path of optic receive "
    echo "     links in an NCS6K multi-chassis system. For all s2 rx and s3 rx "
    echo "     links that are not operationally UP, it creates a tar ball with"
    echo "     debug information from FE1600s, CXPs and CDR devices in the path."
    echo ""
    echo "     Note: Currently only optical links are supported."
}

if [[ $# -ne 3 ]]; then
    echo ""
    echo "Invalid number of arguments!!"
    echo ""
    print_usage;
    exit 2;
fi

stage=$(echo $1 | tr [:upper:] [:lower:])
dir=$(echo $2 | tr [:upper:] [:lower:])
linkhandle=$3
linkhandle_=$(echo $3 | tr '/' '_')
cdrtype=$(echo $4 | tr [:upper:] [:lower:])

if ! [[ "${stage}" =~ (fia|s[123]) ]]; then
    echo ""
    echo "Invalid stage ${stage}!!"
    echo ""

    print_usage
    exit 4
fi

if ! [[ "${dir}" =~ [rt]x ]]; then
    echo ""
    echo "Invalid direction ${dir}!!"
    echo ""

    print_usage
    exit 3
fi

parselink ${linkhandle}
if [ $? -ne 0 ]; then
    echo ""
    echo "Invalid link handle ${linkhandle}!!"
    echo ""

    print_usage
    exit 4
fi


function check_card {
    loc=$1
    show_cmd "show platform location ${loc} | save /tmp/slotinfo" >& /dev/null
    if [[ "$MYADDR" != "$CONFDADDR" ]]; then
        scp -p ${CONFDADDR}:/tmp/slotinfo /tmp >& /dev/null
    fi
    cardstate=$(grep "^${loc}" /tmp/slotinfo | cut -b35-48 | sed -e 's/  *//g')
    cardtype=$(grep "^${loc} .*OPERATIONAL" /tmp/slotinfo | awk '{print $2}' | sed -e 's/  *//')
    rm /tmp/slotinfo
    if [[ "$MYADDR" != "$CONFDADDR" ]]; then
        ssh -q ${CONFDADDR} "rm /tmp/slotinfo"
    fi
}

#
#
# Number of gennum register dumps to be collected one after the other.
#
num_gennum_dumps=2

cleanup() {
    rm -f /root/${this_link__}_*.log
}

trap cleanup SIGHUP SIGTERM SIGINT

tempfile="/root/${linkhandle_}_${stage}_${dir}.log"
[ -e ${tempfile} ] && rm ${tempfile}
show_cmd "show controller fabric link port ${stage} ${dir} ${linkhandle} | i FC | save ${tempfile}" >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
if [[ "$MYADDR" != "$CONFDADDR" ]]; then
    scp -p ${CONFDADDR}:${tempfile} /root >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
    ssh -q ${CONFDADDR} rm ${tempfile} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
fi

if [ ! -s ${tempfile} ]; then
    echo ""
    echo "link ${stage} ${dir} ${linkhandle} does not exist or "
    echo "is an invalid combination of stage, direction and link handle!!"
    echo ""

    print_usage
    exit 5
fi

rm -f $PLATFORM_OUTPUT
show_cmd "show platform"  > $PLATFORM_OUTPUT

IFS=$'\n'
for line in `cat ${tempfile}`;
do
    remotegennum=""
    thisgennum=""
    thiscxpconn=""
    remotecxpconn=""

    #
    # parse the various fields of the 'show controller fabric link port' output
    #
    thislink=$(echo ${line} | cut -b1-15 | sed -e 's/ *//g')
    thisgennum=$(echo ${line} | cut -b26-36 | sed -e 's/ *//g')
    if [ "${stage}" != "fia" ]; then
        remotelink=$(echo ${line} | cut -b67- | sed -e 's/ *//g')
        remotegennum=$(echo ${line} | cut -b56-66 | sed -e 's/ *//g')
        thiscxpconn=$(echo ${line} | cut -b37-44 | sed -e 's/ *//g')
        remotecxpconn=$(echo ${line} | cut -b45-52 | sed -e 's/ *//g')
    else
        remotelink=$(echo ${line} | cut -b36- | sed -e 's/ *//g')
    fi

    this_link__=$(echo $thislink | tr '/' '_')
    remote_link__=$(echo $remotelink | tr '/' '_')

    if [ "${stage}" != "fia" ]; then
        IFS=' ' eval ccc_args=(`eval echo $thiscxpconn | tr '/' ' '`)
        thiscxpport=$(echo ${ccc_args[0]} | sed -e 's/ *//g' -e 's/0\([0-9]\)/\1/')
        thiscxpfiber=$(echo ${ccc_args[1]} | sed -e 's/ *//g' -e 's/0\([0-9]\)/\1/')
    
        IFS=' ' eval ccc_args=(`echo $remotecxpconn | tr '/' ' '`)
        remotecxpport=$(echo ${ccc_args[0]} | sed -e 's/ *//g' -e 's/0\([0-9]\)/\1/')
        remotecxpfiber=$(echo ${ccc_args[1]} | sed -e 's/ *//g' -e 's/0\([0-9]\)/\1/')
    fi

    thisnode=$(parselink -d ${thislink})
    thisnodeip=$(parselink -i ${thislink})
    thisphys=$(parselink -f ${thislink})
    thisrack=$(parselink -r ${thislink})
    thisslot=$(parselink -s ${thislink})
    thissfi=$(parselink -p ${thislink})
    thisinst=$(parselink -a ${thislink})
    thisunit=$(parselink -u ${thislink})
    thislinknum=$(parselink -l ${thislink})

    remotenode=$(parselink -d ${remotelink})
    remotenodeip=$(parselink -i ${remotelink})
    remotephys=$(parselink -f ${remotelink})
    remoterack=$(parselink -r ${remotelink})
    remoteslot=$(parselink -s ${remotelink})
    remotesfi=$(parselink -p ${remotelink})
    remoteinst=$(parselink -a ${remotelink})
    remoteunit=$(parselink -u ${remotelink})
    remotelinknum=$(parselink -l ${remotelink})

    rm -f /root/${this_link__}_*.log >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
    datestring=$(date | tr ' :' '-')

    filename="/root/${this_link__}_${datestring}_README.log"
    echo "##############################################################################################" > ${filename}
    echo "${line}" >> ${filename}
    echo "##############################################################################################" >> ${filename}

    echo "Starting to dump data for failing link ${thislink}..."

    if [[ ${thisslot} =~ FC ]]; then
        echo "Saving local SFE dsc output in ${this_link__}_${datestring}_sfe_${this_link__}_dsc.log" >> ${filename}
        show_cmd "show controller sfe diagshell ${thisinst} \"phy diag sfi${thissfi} dsc\" location ${thisrack}/${thisslot} | save /root/${this_link__}_${datestring}_sfe_${this_link__}_dsc.log" >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        echo "Saving local SFE diag link output in ${this_link__}_${datestring}_sfe_${this_link__}_diag_link.log" >> ${filename}
        show_cmd "show controller sfe diagshell ${thisinst} \"fab link ${thissfi}\" location ${thisrack}/${thisslot} | save /root/${this_link__}_${datestring}_sfe_${this_link__}_diag_link.log" >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        echo "Saving local SFE ps output in ${this_link__}_${datestring}_sfe_${this_link__}_ps.log" >> ${filename}
        show_cmd "show controller sfe diagshell ${thisinst} \"ps sfi${thissfi}\" location ${thisrack}/${thisslot} | save /root/${this_link__}_${datestring}_sfe_${this_link__}_ps.log" >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
    
        echo "Saving local sfe_driver trace in ${this_link__}_${datestring}_sfe_driver_${this_link__}.log" >> ${filename}
        if [ "$MYADDR" != "$thisnodeip" ]; then
            ssh ${thisnodeip} "/opt/cisco/calvados/bin/ctracedec -ktgfspTlw sfe_driver | grep \"\(unit $thisunit link $thislinknum\>\|sfi${thislinknum}\>\)\" | grep -v conf_pend > /root/${this_link__}_${datestring}_sfe_driver_${this_link__}.log"
            scp ${thisnodeip}:/root/${this_link__}_${datestring}_sfe_driver_${this_link__}.log /root
            ssh -q ${thisnodeip} rm /root/${this_link__}_${datestring}_sfe_driver_${this_link__}.log
        else
            /opt/cisco/calvados/bin/ctracedec -ktgfspTlw sfe_driver | grep "\(unit $thisunit link $thislinknum\>\|sfi${thislinknum}\>\)" | grep -v conf_pend > /root/${this_link__}_${datestring}_sfe_driver_${this_link__}.log
        fi
       
        echo "Saving local asic errors in ${this_link__}_${datestring}_sfe_${this_link__}_asic_error.log" >> ${filename}
        show_cmd "show asic-error SFE ${thisunit} all location ${thisnode} | save /root/${this_link__}_${datestring}_sfe_${this_link__}_asic_error.log" >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
    else
        rp_xr_ip=$(findnodeip $remotenode "IOS-XR-RP")
        driver_rack_slot=$(parselink -d $thislink | tr '/' '_')
        nodeid=$(ssh -q $rp_xr_ip /pkg/bin/node_conversion -i "/net/node${driver_rack_slot}")
        echo "Saving local FIA dsc output in ${this_link__}_${datestring}_fia_${this_link__}_dsc.log" >> ${filename}
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"log file=${this_link__}_${datestring}_fia_${this_link__}_dsc.log on\" -u ${thisinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"phy diag sfi${thissfi} dsc\" -u ${thisinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"log file=${this_link__}_${datestring}_fia_${this_link__}_dsc.log off\" -u ${thisinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        scp -p $thisnodeip:/${this_link__}_${datestring}_fia_${this_link__}_dsc.log /root
        ssh -q $thisnodeip rm /${this_link__}_${datestring}_fia_${this_link__}_dsc.log

        echo "Saving local FIA diag link output in ${this_link__}_${datestring}_fia_${this_link__}_diag_link.log" >> ${filename}
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"log file=${this_link__}_${datestring}_fia_${this_link__}_diag_link.log on\" -u ${thisinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"fab link ${thislinknum}\" -u ${thisinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"log file=${this_link__}_${datestring}_fia_${this_link__}_diag_link.log off\" -u ${thisinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        scp -p $thisnodeip:/${this_link__}_${datestring}_fia_${this_link__}_diag_link.log /root
        ssh -q $thisnodeip rm /${this_link__}_${datestring}_fia_${this_link__}_diag_link.log

        echo "Saving local FIA ps output in ${this_link__}_${datestring}_fia_${this_link__}_ps.log" >> ${filename}
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"log file=${this_link__}_${datestring}_fia_${this_link__}_ps.log on\" -u ${thisinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c ps -u ${thisinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"log file=${this_link__}_${datestring}_fia_${this_link__}_ps.log off\" -u ${thisinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        scp -p $thisnodeip:/${this_link__}_${datestring}_fia_${this_link__}_ps.log /root
        ssh -q $thisnodeip rm /${this_link__}_${datestring}_fia_${this_link__}_ps.log

        echo "Saving local fia_driver trace in ${this_link__}_${datestring}_fia_driver_${this_link__}.log" >> ${filename}
        ssh ${thisnodeip} "/pkg/bin/fia_show_ltrace -A > /root/${this_link__}_${datestring}_fia_driver_${this_link__}.log"
        scp -p ${thisnodeip}:/root/${this_link__}_${datestring}_fia_driver_${this_link__}.log /root
        ssh -q ${thisnodeip} rm /root/${this_link__}_${datestring}_fia_driver_${this_link__}.log

        echo "Saving local asic errors in ${this_link__}_${datestring}_fia_${this_link__}_asic_error.log" >> ${filename}
        ssh -q $rp_xr_ip "/pkg/bin/asic_errors_show -n ${nodeid} -a 0x4 -i ${thisinst}  -e 0x7 -s -t 0x0 -m 0x0" > /root/${this_link__}_${datestring}_fia_${this_link__}_asic_error.log
    fi

    if [ -n "$thisgennum" ]; then
        IFS=' ' eval ccc_args=(`echo $thisgennum | tr '-' ' ' | tr [:upper:] [:lower:]`)
        slice=${ccc_args[0]}
        slice=$(echo ${slice} | sed -e 's/ *//g')
        port=${ccc_args[1]}
        port=$(echo ${port} | sed -e 's/ *//g' -e 's/0\([0-9]\)/\1/')
        lane=${ccc_args[2]}
        lane=$(echo ${lane} | sed -e 's/ *//g' -e 's/0\([0-9]\)/\1/')

        check_card "${thisrack}/${thisslot}"



        if [ "$MYADDR" != "$thisnodeip" ]; then
            if [[ ${thisslot} =~ FC ]]; then
                if [ "$cardtype" == "NCS-F-FC2" ] || [ "$cardtype" == "NCS4KF-FC2-C" ] ; then
                    #                                            #
                    #    Rigel/Ungoliant Board. Has the new BCM82212 CDR   #
                    #                                            #
                    cdrdevice=$((slice * 8 + port))
                    echo "Saving local BCM82212 status in ${this_link__}_${datestring}_bcm82212_${thisrack}_${thisslot}_${slice}_${port}_status.log" >> ${filename}
                    ssh  -q ${thisnodeip} /opt/cisco/calvados/sbin/cdr_bcm82212_ccc_client --cdr_status -c$cdrdevice -s${thisphys} > /root/${this_link__}_${datestring}_bcm82212_${thisrack}_${thisslot}_${slice}_${port}_status.log
                else
                    echo "Saving local gennum context in ${this_link__}_${datestring}_gn241x_${thisrack}_${thisslot}_${slice}_${port}_ctx.log" >> ${filename}
                    ssh  -q ${thisnodeip} /opt/cisco/calvados/sbin/lc_fab_ccc_plugin_cmd -c ctx -$slice -$port -L${thisphys} > /root/${this_link__}_${datestring}_gn241x_${thisrack}_${thisslot}_${slice}_${port}_ctx.log
                    for (( iter = 1; iter <= ${num_gennum_dumps} ; iter++ )); do
                        echo "Saving local gennum register dump ${iter} in ${this_link__}_${datestring}_gn241x_${thisrack}_${thisslot}_${slice}_${port}_reg_dump_${iter}.log" >> ${filename}
                        ssh  -q ${thisnodeip} /opt/cisco/calvados/sbin/lc_fab_ccc_plugin_cmd -c read -$slice -$port -L${thisphys} -r 0x0 -z32768 > /root/${this_link__}_${datestring}_gn241x_${thisrack}_${thisslot}_${slice}_${port}_reg_dump_${iter}.log
                    done
                fi
            else
                thisnode_rack_slot=${thisnode%/CPU0}
                thisnode_calvados_ip=$(findnodeip ${thisnode_rack_slot} "sysadmin")
                ssh  -q ${thisnode_calvados_ip} "/opt/cisco/calvados/sbin/lc_fab_ccc_plugin_cmd -c ctx -$slice -$port" > /root/${this_link__}_${datestring}_gn241x_${thisrack}_${thisslot}_${slice}_${port}_ctx.log
                for (( iter = 1; iter <= ${num_gennum_dumps} ; iter++ )); do
                    echo "Saving local gennum register dump ${iter} in ${this_link__}_${datestring}_gn241x_${thisrack}_${thisslot}_${slice}_${port}_reg_dump_${iter}.log" >> ${filename}
                    ssh  -q ${thisnode_calvados_ip} "/opt/cisco/calvados/sbin/lc_fab_ccc_plugin_cmd -c read -$slice -$port -r 0x0 -z32768" > /root/${this_link__}_${datestring}_gn241x_${thisrack}_${thisslot}_${slice}_${port}_reg_dump_${iter}.log
                done
            fi
        else
            if [ "$cardtype" == "NCS-F-FC2" ] || [ "$cardtype" == "NCS4KF-FC2-C" ] ; then
                cdrdevice=$((slice * 8 + port))
                echo "Saving local BCM82212 status in ${this_link__}_${datestring}_bcm82212_${thisrack}_${thisslot}_${slice}_${port}_status.log" >> ${filename}
                /opt/cisco/calvados/sbin/cdr_bcm82212_ccc_client --cdr_status -c$cdrdevice -s${thisphys} > /root/${this_link__}_${datestring}_bcm82212_${thisrack}_${thisslot}_${slice}_${port}_status.log
            else
                /opt/cisco/calvados/sbin/lc_fab_ccc_plugin_cmd -c ctx -$slice -$port -L${thisphys} > /root/${this_link__}_${datestring}_gn241x_${thisrack}_${thisslot}_${slice}_${port}_ctx.log
                for (( iter = 1; iter <= ${num_gennum_dumps} ; iter++ )); do
                    echo "Saving local gennum register dump ${iter} in ${this_link__}_${datestring}_gn241x_${thisrack}_${thisslot}_${slice}_${port}_reg_dump_${iter}.log" >> ${filename}
                    /opt/cisco/calvados/sbin/lc_fab_ccc_plugin_cmd -c read -$slice -$port -L${thisphys} -r 0x0 -z32768 > /root/${this_link__}_${datestring}_gn241x_${thisrack}_${thisslot}_${slice}_${port}_reg_dump_${iter}.log
                done
            fi
        fi
    fi

    if [ -n "$thiscxpconn" ]; then
        echo "Saving local cxp dom output in ${this_link__}_${datestring}_cxp_${thisrack}_${thisslot}_${thiscxpport}_dom.log" >> ${filename}
        show_cmd "show controller fabric cxp dom location ${thisrack}/${thisslot} port ${thiscxpport} | save /root/${this_link__}_${datestring}_cxp_${thisrack}_${thisslot}_${thiscxpport}_dom.log" >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
    
        echo "Saving local cxp register dump in ${this_link__}_${datestring}_cxp_${thisrack}_${thisslot}_${thiscxpport}_reg_dump.log" >> ${filename}
        if [ "$MYADDR" != "$thisnodeip" ]; then
            ssh  -q ${thisnodeip} /opt/cisco/calvados/sbin/fc_cxp_client -c read -p${thiscxpport} -L${thisphys} -r 0x0 -z384 > /root/${this_link__}_${datestring}_cxp_${thisrack}_${thisslot}_${thiscxpport}_reg_dump.log
            ssh  -q ${thisnodeip} /opt/cisco/calvados/sbin/fc_cxp_client -c read -p${thiscxpport} -L${thisphys} -r 0x0 -z384 -R >> /root/${this_link__}_${datestring}_cxp_${thisrack}_${thisslot}_${thiscxpport}_reg_dump.log
        else
            /opt/cisco/calvados/sbin/fc_cxp_client -c read -p${thiscxpport} -L${thisphys} -r 0x0 -z384 > /root/${this_link__}_${datestring}_cxp_${thisrack}_${thisslot}_${thiscxpport}_reg_dump.log
            /opt/cisco/calvados/sbin/fc_cxp_client -c read -p${thiscxpport} -L${thisphys} -r 0x0 -z384 -R >> /root/${this_link__}_${datestring}_cxp_${thisrack}_${thisslot}_${thiscxpport}_reg_dump.log
        fi
    
        echo "Saving cxp power check output in ${this_link__}_${datestring}_cxppower.log" >> ${filename}
        cxppowerchk "${thisrack}/${thisslot}" "${thiscxpport}"  "${remoterack}/${remoteslot}" "${remotecxpport}" > /root/${this_link__}_${datestring}_cxppower.log
    
        echo "Saving remote cxp dom output in ${this_link__}_${datestring}_cxp_${remoterack}_${remoteslot}_${remotecxpport}_reg_dump.log" >> ${filename}
        if [ "$MYADDR" != "$remotenodeip" ]; then
            ssh  -q ${remotenodeip} /opt/cisco/calvados/sbin/fc_cxp_client -c read -p${remotecxpport} -L${remotephys} -r 0x0 -z384 > /root/${this_link__}_${datestring}_cxp_${remoterack}_${remoteslot}_${remotecxpport}_reg_dump.log
            ssh  -q ${remotenodeip} /opt/cisco/calvados/sbin/fc_cxp_client -c read -p${remotecxpport} -L${remotephys} -r 0x0 -z384 -R >> /root/${this_link__}_${datestring}_cxp_${remoterack}_${remoteslot}_${remotecxpport}_reg_dump.log
        else
            /opt/cisco/calvados/sbin/fc_cxp_client -c read -p${remotecxpport} -L${remotephys} -r 0x0 -z384 > /root/${this_link__}_${datestring}_cxp_${remoterack}_${remoteslot}_${remotecxpport}_reg_dump.log
            /opt/cisco/calvados/sbin/fc_cxp_client -c read -p${remotecxpport} -L${remotephys} -r 0x0 -z384 -R >> /root/${this_link__}_${datestring}_cxp_${remoterack}_${remoteslot}_${remotecxpport}_reg_dump.log
        fi
    
        echo "Saving remote cxp dom output in ${this_link__}_${datestring}_cxp_${remoterack}_${remoteslot}_${remotecxpport}_dom.log" >> ${filename}
        show_cmd "show controller fabric cxp dom location ${remoterack}/${remoteslot} port ${remotecxpport} | save /root/${this_link__}_${datestring}_cxp_${remoterack}_${remoteslot}_${remotecxpport}_dom.log" >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
    fi

        check_card "${remoterack}/${remoteslot}"




    if [ -n "$remotegennum" ]; then
        IFS=' ' eval ccc_args=(`echo $remotegennum | tr '-' ' ' | tr [:upper:] [:lower:]`)
        slice=${ccc_args[0]}
        slice=$(echo ${slice} | sed -e 's/ *//g')
        port=${ccc_args[1]}
        port=$(echo ${port} | sed -e 's/ *//g' -e 's/0\([0-9]\)/\1/')
        lane=${ccc_args[2]}
        lane=$(echo ${lane} | sed -e 's/ *//g' -e 's/0\([0-9]\)/\1/')

        echo "Saving remote gennum context in ${this_link__}_${datestring}_gn241x_${remoterack}_${remoteslot}_${slice}_${port}_ctx.log" >> ${filename}

        if [ "$MYADDR" != "$remotenodeip" ]; then
            if [[ ${remoteslot} =~ FC ]]; then
                if [ "$cardtype" == "NCS-F-FC2" ] || [ "$cardtype" == "NCS4KF-FC2-C" ] ; then
                    #                                            #
                    #    Rigel/Ungoliant Board. Has the new BCM82212 CDR   #
                    #                                            #
                    cdrdevice=$((slice * 8 + port))
                    echo "Saving local BCM82212 status in ${this_link__}_${datestring}_bcm82212_${remoterack}_${remoteslot}_${slice}_${port}_status.log" >> ${filename}
                    ssh  -q ${remotenodeip} /opt/cisco/calvados/sbin/cdr_bcm82212_ccc_client --cdr_status -c$cdrdevice -s${remotephys} > /root/${this_link__}_${datestring}_bcm82212_${remoterack}_${remoteslot}_${slice}_${port}_status.log
                else
                    ssh  -q ${remotenodeip} /opt/cisco/calvados/sbin/lc_fab_ccc_plugin_cmd -c ctx -$slice -$port -L${remotephys} > /root/${this_link__}_${datestring}_gn241x_${remoterack}_${remoteslot}_${slice}_${port}_ctx.log
        
                    for (( iter = 1; iter <= ${num_gennum_dumps} ; iter++ )); do
                        echo "Saving remote gennum register dump ${iter} in ${this_link__}_${datestring}_gn241x_${remoterack}_${remoteslot}_${slice}_${port}_reg_dump_${iter}.log" >> ${filename}
                        ssh  -q ${remotenodeip} /opt/cisco/calvados/sbin/lc_fab_ccc_plugin_cmd -c read -$slice -$port -L${remotephys} -r 0x0 -z32768 > /root/${this_link__}_${datestring}_gn241x_${remoterack}_${remoteslot}_${slice}_${port}_reg_dump_${iter}.log
                    done
                fi
            else
                remotenode_rack_slot=${remotenode%/CPU0}
                remotenode_calvados_ip=$(findnodeip ${remotenode_rack_slot} "sysadmin")
                ssh  -q ${remotenode_calvados_ip} "/opt/cisco/calvados/sbin/lc_fab_ccc_plugin_cmd -c ctx -$slice -$port" > /root/${this_link__}_${datestring}_gn241x_${remoterack}_${remoteslot}_${slice}_${port}_ctx.log
                for (( iter = 1; iter <= ${num_gennum_dumps} ; iter++ )); do
                    echo "Saving local gennum register dump ${iter} in ${this_link__}_${datestring}_gn241x_${remoterack}_${remoteslot}_${slice}_${port}_reg_dump_${iter}.log" >> ${filename}
                    ssh  -q ${remotenode_calvados_ip} "/opt/cisco/calvados/sbin/lc_fab_ccc_plugin_cmd -c read -$slice -$port -r 0x0 -z32768" > /root/${this_link__}_${datestring}_gn241x_${remoterack}_${remoteslot}_${slice}_${port}_reg_dump_${iter}.log
                done
            fi
        else
            if [ "$cardtype" == "NCS-F-FC2" ] || [ "$cardtype" == "NCS4KF-FC2-C" ] ; then
                #                                            #
                #    Rigel/Ungoliant Board. Has the new BCM82212 CDR   #
                #                                            #
                cdrdevice=$((slice * 8 + port))
                echo "Saving local BCM82212 status in ${this_link__}_${datestring}_bcm82212_${remoterack}_${remoteslot}_${slice}_${port}_status.log" >> ${filename}
                /opt/cisco/calvados/sbin/cdr_bcm82212_ccc_client --cdr_status -c$cdrdevice -s${remotephys} > /root/${this_link__}_${datestring}_bcm82212_${remoterack}_${remoteslot}_${slice}_${port}_status.log
            else
                /opt/cisco/calvados/sbin/lc_fab_ccc_plugin_cmd -c ctx -$slice -$port -L${remotephys} > /root/${this_link__}_${datestring}_gn241x_${remoterack}_${remoteslot}_${slice}_${port}_ctx.log
                for (( iter = 1; iter <= ${num_gennum_dumps} ; iter++ )); do
                    echo "Saving remote gennum register dump ${iter} in ${this_link__}_${datestring}_gn241x_${remoterack}_${remoteslot}_${slice}_${port}_reg_dump_${iter}.log" >> ${filename}
                    /opt/cisco/calvados/sbin/lc_fab_ccc_plugin_cmd -c read -$slice -$port -L${remotephys} -r 0x0 -z32768 > /root/${this_link__}_${datestring}_gn241x_${remoterack}_${remoteslot}_${slice}_${port}_reg_dump_${iter}.log
                done
            fi
        fi
    fi

    if [[ ${remoteslot} =~ FC ]]; then
        echo "Saving remote SFE dsc output in ${this_link__}_${datestring}_sfe_${remote_link__}_dsc.log" >> ${filename}
        show_cmd "show controller sfe diagshell ${remoteinst} \"phy diag sfi${remotesfi} dsc\" location ${remoterack}/${remoteslot} | save /root/${this_link__}_${datestring}_sfe_${remote_link__}_dsc.log" >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        echo "Saving remote SFE diag link output in ${this_link__}_${datestring}_sfe_${remote_link__}_diag_link.log" >> ${filename}
        show_cmd "show controller sfe diagshell ${remoteinst} \"fab link ${remotesfi}\" location ${remoterack}/${remoteslot} | save /root/${this_link__}_${datestring}_sfe_${remote_link__}_diag_link.log" >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        echo "Saving remote SFE ps output in ${this_link__}_${datestring}_sfe_${remote_link__}_ps.log" >> ${filename}
        show_cmd "show controller sfe diagshell ${remoteinst} \"ps sfi${remotesfi}\" location ${remoterack}/${remoteslot} | save /root/${this_link__}_${datestring}_sfe_${remote_link__}_ps.log" >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
    
        echo "Saving remote sfe_driver trace in ${this_link__}_${datestring}_sfe_driver_${remote_link__}.log" >> ${filename}
        if [ "$MYADDR" != "$remotenodeip" ]; then
            ssh ${remotenodeip} "/opt/cisco/calvados/bin/ctracedec -ktgfspTlw sfe_driver | grep \"\(unit $remoteunit link $remotelinknum\>\|sfi${remotelinknum}\>\)\" | grep -v conf_pend> /root/${this_link__}_${datestring}_sfe_driver_${remote_link__}.log"
            scp ${remotenodeip}:/root/${this_link__}_${datestring}_sfe_driver_${remote_link__}.log /root
            ssh -q ${remotenodeip} rm /root/${this_link__}_${datestring}_sfe_driver_${remote_link__}.log
        else
            /opt/cisco/calvados/bin/ctracedec -ktgfspTlw sfe_driver | grep "\(unit $remoteunit link $remotelinknum\>\|sfi${remotelinknum}\>\)" | grep -v conf_pend > /root/${this_link__}_${datestring}_sfe_driver_${remote_link__}.log
        fi
        
        echo "Saving remote asic errors in ${this_link__}_${datestring}_sfe_${remote_link__}_asic_error.log" >> ${filename}
        show_cmd "show asic-error SFE ${remoteunit} all location ${remotenode} | save /root/${this_link__}_${datestring}_sfe_${remote_link__}_asic_error.log" >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
    else
        rp_xr_ip=$(findnodeip $thisnode "IOS-XR-RP")
        driver_rack_slot=$(parselink -d $remotelink | tr '/' '_')
        nodeid=$(ssh -q $rp_xr_ip /pkg/bin/node_conversion -i "/net/node${driver_rack_slot}")
        echo "Saving remote FIA dsc output in ${this_link__}_${datestring}_fia_${remote_link__}_dsc.log" >> ${filename}
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"log file=${this_link__}_${datestring}_fia_${remote_link__}_dsc.log on\" -u ${remoteinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"phy diag sfi${remotesfi} dsc\" -u ${remoteinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"log file=${this_link__}_${datestring}_fia_${remote_link__}_dsc.log off\" -u ${remoteinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        scp -p $remotenodeip:/${this_link__}_${datestring}_fia_${remote_link__}_dsc.log /root
        ssh -q $remotenodeip rm /${this_link__}_${datestring}_fia_${remote_link__}_dsc.log

        echo "Saving remote FIA diag link output in ${this_link__}_${datestring}_fia_${remote_link__}_diag_link.log" >> ${filename}
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"log file=${this_link__}_${datestring}_fia_${remote_link__}_diag_link.log on\" -u ${remoteinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"fab link ${remotelinknum}\" -u ${remoteinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"log file=${this_link__}_${datestring}_fia_${remote_link__}_diag_link.log off\" -u ${remoteinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        scp -p $remotenodeip:/${this_link__}_${datestring}_fia_${remote_link__}_diag_link.log /root
        ssh -q $remotenodeip rm /${this_link__}_${datestring}_fia_${remote_link__}_diag_link.log

        echo "Saving remote FIA ps output in ${this_link__}_${datestring}_fia_${remote_link__}_ps.log" >> ${filename}
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"log file=${this_link__}_${datestring}_fia_${remote_link__}_ps.log on\" -u ${remoteinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c ps -u ${remoteinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        ssh -q $rp_xr_ip /pkg/bin/fia_driver_show -c \"log file=${this_link__}_${datestring}_fia_${remote_link__}_ps.log off\" -u ${remoteinst} -n ${nodeid} >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        scp -p $remotenodeip:/${this_link__}_${datestring}_fia_${remote_link__}_ps.log /root
        ssh -q $remotenodeip rm /${this_link__}_${datestring}_fia_${remote_link__}_ps.log

        echo "Saving remote fia_driver trace in ${this_link__}_${datestring}_fia_driver_${remote_link__}.log" >> ${filename}
        ssh ${remotenodeip} "/pkg/bin/fia_show_ltrace -A > /root/${this_link__}_${datestring}_fia_driver_${remote_link__}.log"
        scp -p ${remotenodeip}:/root/${this_link__}_${datestring}_fia_driver_${remote_link__}.log /root
        ssh -q ${remotenodeip} rm /root/${this_link__}_${datestring}_fia_driver_${remote_link__}.log

        echo "Saving remote asic errors in ${this_link__}_${datestring}_fia_${remote_link__}_asic_error.log" >> ${filename}
        ssh -q $rp_xr_ip "/pkg/bin/asic_errors_show -n ${nodeid} -a 0x4 -i ${remoteinst}  -e 0x7 -s -t 0x0 -m 0x0" > /root/${this_link__}_${datestring}_fia_${remote_link__}_asic_error.log
    fi

    if [[ "$MYADDR" != "$CONFDADDR" ]]; then
        scp -p ${CONFDADDR}:/root/${this_link__}_${datestring}_*.log /root >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
        ssh -q ${CONFDADDR} rm /root/${this_link__}_${datestring}_*.log >& /tmp/cmd.out.$$; mv /tmp/cmd.out.$$  /root/${linkhandle_}_command.log
    fi

    # remove any older log for the same link.
    rm -f /root/perm_shut_${this_link__}_*.tgz

    tar cvfz /root/perm_shut_${this_link__}_${datestring}_debug.tgz /root/${this_link__}_*.log --no-recursion
    rm /root/${this_link__}_*.log

    ## if there are too many dump created, keep the most recent 20 
    ## and remove the older ones.
    filecount=$(ls -ltr /root/perm_shut_*.tgz | wc -l)
    while [ $filecount -gt 20 ]; do
        oldestfile=$(ls -ltr /root/perm_shut_*.tgz | head -1 | awk '{print $NF}')
        rm ${oldestfile}
        filecount=$(ls -ltr /root/perm_shut_*.tgz | wc -l)
    done

    echo "Done dumping data for failing link ${thislink}..."
done

