#!/bin/bash
#
# A platform hw profile allows the memory and CPU values of a domain to be
# modified. A domain here is an LXC domain, i.e. Calvados, XR or UVF.
#
# The hw profile can be set in the linux boot command line via __hw_profile
# If not set, PLATFORM_DEFAULT_HW_PROFILE will be used.
#
# To calculate the memory and CPU requirements for each domain, a number of
# steps are then followed in order:
#
# 1. Global defaults for all hw profiles are established in this file
#    (hw_profiles_functions) via:
#
#        platform_hw_profile_calvados_get_defaults
#        platform_hw_profile_xr_get_defaults
#        platform_hw_profile_uvf_get_defaults
#        platform_hw_profile_lcp_get_defaults
#
#    Which set
#
#        PLATFORM_CALVADOS_VM_MEM_MB
#        PLATFORM_XR_VM_MEM_MB
#        PLATFORM_XR_PACKET_MEM_MB
#        PLATFORM_LCP_VM_MEM_MB
#
#        PLATFORM_CALVADOS_VM_CPU
#        PLATFORM_CP_DP_CORE_ASSIGN
#
# 2. A database (platform_hw_profiles.sh) with hw profile specific information
#    is then read by get_hw_profile.sh. This will override the above values.
#
#    The following functions retrieve the information from the database:
#
#        platform_hw_profile_calvados_get
#        platform_hw_profile_xr_get
#        platform_hw_profile_uvf_get
#        platform_hw_profile_lcp_get
#
# 3. A profile specific script is sourced to allow more complex requirments
#    e.g. /etc/init.d/vpe.profile which must define:
#
#        platform_hw_profile_patch_hostos_override
#        platform_hw_profile_patch_calvados_override
#        function platform_hw_profile_patch_uvf_override
#
# 4. For fine granularity the end user can specify the:
#
#        __hw_profile_vm_mem_gb 
#        __hw_profile_packet_mem_mb 
#        __hw_profile_cpu 
#
#    command line boot values to override the defaults.
#
# 5. Some values then need to be converted. e.g. from Gb to Kb for certain
#    domains.
#
#        platform_hw_profile_calvados_fixup
#
# 6. The patch functions are called with the final values from above
#
#        platform_hw_profile_patch_host        - to patch calvados startup
#        platform_hw_profile_patch_calvados    - to patch XR startup
#        platform_hw_profile_patch_uvf         - to patch UVF startup
#
# Finally we will source a profile specific shell script
# Neil McGill
#
# Copyright (c) 2014-2018, 2021 by Cisco Systems, Inc.
# All rights reserved.
#

#
# Calvados and UVF XML want memory settings in terms of Kb
#
MB_TO_KB_CONVERT=1024

cgroup_mount_pt="/dev/cgroup"
KTHREAD_PID=2

#
# Check if we're running on a ThinXR system
#
function am_i_thinxr()
{
    [[ -e "/opt/cisco/thinxr/am_i_thinxr" ]]
}

#
# Dig our the cmdline from the various places it may be hiding
#
function find_cmdline
{
    local rootdir=$1

    for cmdline in $rootdir/proc/cmdline $rootdir/root/cmdline /proc/cmdline /root/cmdline
    do
        if [[ -f $cmdline ]]; then
            return
        fi
    done

    cmdline=
}

#
# Find a file from common paths
#
function find_file()
{
    local file=$1

    for dir in  \
                $rootdir/etc/rc.d/init.d  \
                /etc/rc.d/init.d  \
                .  ../hw_profiles ../scripts
    do
        if [[ -f $dir/$file ]]; then
            echo $dir/$file
            return
        fi
    done
}

source_hw_profiles()
{
    local file=

    for file in pd-functions \
                hw_profiles_functions \
                calvados_bootstrap.cfg \
                platform_hw_profiles.sh
    do
        local found=`find_file $file`
        if [[ "$found" = "" ]]; then
            platform_log_error "Could not find $file to source"
        fi

        source $found
    done
}

#
# set cmdline utilities
#
GRUB_OPEN_CNT=0

function get_cmdline()
{
    local cmdline_file=$1
    local arg_name=$2
    local old_value=`grep -o " $arg_name=[^ ]*" $cmdline_file | awk -F= '{print $2}'`

    printf "$old_value"
}

function set_cmdline()
{
    local cmdline_file=$1
    local arg_name=$2
    local arg_value=$3
    local old_value=`grep -o " $arg_name=[^ ]*" $cmdline_file | awk -F= '{print $2}'`

    if [[ "$arg_value" != "" ]]; then
        if [[ "$old_value" != "" ]]; then
            #set cmdline
            sed -i "s~ $arg_name=[^ ]*~ $arg_name=$arg_value~g" $cmdline_file
        else
            #add cmdline
            sed -i "s~\(platform=xrv9k\)~\1 $arg_name=$arg_value~g" $cmdline_file
        fi
    else
        sed -i "s/ $arg_name=[^ ]*//g" $cmdline_file
    fi
}

function open_grub_cmdline()
{
    if (( $GRUB_OPEN_CNT == 0 )); then
        TMPMNT=`mktemp -d`
        if [[ -b /dev/vda1 ]]; then
            mount /dev/vda1 $TMPMNT
        elif [[ -b /dev/sda1 ]]; then
            mount /dev/sda1 $TMPMNT
        elif [[ -b /dev/nvme0n1p1 ]]; then
            mount /dev/nvme0n1p1 $TMPMNT
        else
            platform_log_error "cannot find boot disk partition"
            rmdir $TMPMNT
            exit -1
        fi
    fi

    GRUB_OPEN_CNT=$(( $GRUB_OPEN_CNT + 1 ))
}

function close_grub_cmdline()
{
    GRUB_OPEN_CNT=$(( $GRUB_OPEN_CNT - 1 ))

    if (( $GRUB_OPEN_CNT == 0 )); then
        if [[ -e "$TMPMNT" ]] ; then
            umount $TMPMNT
            rmdir $TMPMNT
            TMPMNT=
        else
            platform_log_error "cannot find mounted boot disk folder"
        fi
    fi
}

#
# Print domain info
#
function platform_hw_profile_calvados_info
{
    if [[ "$XRV9K_BAREMETAL" = "vrr" ]]; then
        platform_log_console "XRv9K Appliance"
    elif [[ "$XRV9K_BAREMETAL" = "vpe" ]]; then
        platform_log_console "XRv9K vPE Appliance"
    fi
    platform_log_console "Hardware profile: $PLATFORM_HW_PROFILE"

    local HOST_NCPUS=`cat /proc/cpuinfo | grep processor | wc -l`
    local HOST_MEM_MB=`cat /proc/meminfo | grep MemTotal | awk '{print int($2 / 1024)}'`
    local HOST_MEM_GB=$(echo | awk "{printf(\"%2.2f\n\", $HOST_MEM_MB / 1024)}")

    platform_log_console "Host has ${HOST_MEM_GB}GB RAM / $HOST_NCPUS vCPUs"
    platform_log_console \
        "Management plane: ${PLATFORM_CALVADOS_VM_MEM_MB}MB RAM"
}

#
# Print domain info
#
function platform_hw_profile_xr_info
{
    platform_log_console \
        "XR control plane: ${PLATFORM_XR_VM_MEM_MB}MB RAM"
    platform_log_console \
        "XR packet memory: ${PLATFORM_XR_PACKET_MEM_MB}MB RAM"
}

#
# Print domain info
#
function platform_hw_profile_uvf_info
{
    if ! is_distributed; then
        platform_log_console "Centralized LC: ${PLATFORM_LCP_VM_MEM_MB}MB RAM"
    fi
    platform_log_console "Data plane core assignment: $dp_cores"
    platform_log_console "Control plane core assignment: $cp_cores"
}

#
# Print TPA info if configured through meta-data
#
function platform_hw_profile_tpa_info
{
    if tpa_can_modify_host_ncpus; then
        platform_log_console "Third party application core assignment: $APP_HOST_CPUS"
    fi

    if tpa_can_modify_host_memory; then
        platform_log_console "Third party application memory: ${APP_HOST_MEM_GB}GB RAM"
    fi

    if tpa_can_modify_numa_nodes; then
        platform_log_console "Third party NUMA nodes: $APP_HOST_CPUSET_MEMS"
    fi
}

#
# We should know the platform profile by this point and can now configure the
# tool prior to calling it.
#
function platform_hw_profile_init_get
{
    local get_hw_profile="get_hw_profile.sh"
    local found=`find_file $get_hw_profile`

    PLATFORM_HW_PROFILE_TOOL=

    if [[ "$found" = "" ]]; then
        platform_log_error "$get_hw_profile not found"
        return
    fi

    PLATFORM_HW_PROFILE_TOOL="$found -hw-profile $PLATFORM_HW_PROFILE"
}

#
# Get the domain settings from the database. If set already, then ignore the
# values in the database.
#
function platform_hw_profile_calvados_get
{
    #
    # Get the VM memory
    #
    val=`$PLATFORM_HW_PROFILE_TOOL -domain calvados -mem`
    if [[ $? -ne 0 ]]; then
        $PLATFORM_HW_PROFILE_TOOL -domain calvados -mem
        return
    fi

    if [[ "$val" != "" ]]; then
        PLATFORM_CALVADOS_VM_MEM_MB=$(echo | awk "{printf(\"%d\n\", $val * 1024)}")
    fi

    #
    # Get the number of vCPUS
    #
    val=`$PLATFORM_HW_PROFILE_TOOL -domain calvados -cpu`
    if [[ $? -ne 0 ]]; then
        $PLATFORM_HW_PROFILE_TOOL -domain calvados -cpu
        return
    fi

    if [[ "$val" != "" ]]; then
        PLATFORM_CALVADOS_VM_CPU=$val
    fi
}

#
# Get the domain settings from the database. If set already, then ignore the
# values in the database.
#
function platform_hw_profile_xr_get
{
    #
    # Get the VM memory
    #
    val=`$PLATFORM_HW_PROFILE_TOOL -domain xr -mem`
    if [[ $? -ne 0 ]]; then
        $PLATFORM_HW_PROFILE_TOOL -domain xr -mem
        return
    fi

    if [[ "$val" != "" ]]; then
        PLATFORM_XR_VM_MEM_MB=$(echo | awk "{printf(\"%d\n\", $val * 1024)}")
    fi

    #
    # Get the packet memory
    #
    val=`$PLATFORM_HW_PROFILE_TOOL -domain xr -packet-mem`
    if [[ $? -ne 0 ]]; then
        $PLATFORM_HW_PROFILE_TOOL -domain xr -packet-mem
        return
    fi

    if [[ "$val" != "" ]]; then
        PLATFORM_XR_PACKET_MEM_MB=$val
    fi

}

#
# Get the domain settings from the database based on given number of cpus. 
#
function platform_hw_profile_default_uvf_get
{
    local hostncpu=$1
    if [[ "$hostncpu" = "" ]]; then
        return
    fi
    #
    # Get the number of vCPUS
    #
    val=`$PLATFORM_HW_PROFILE_TOOL -domain uvf -cpu -hostncpu $hostncpu`
    if [[ $? -ne 0 ]]; then
        $PLATFORM_HW_PROFILE_TOOL -domain uvf -cpu -hostncpu $hostncpu
    else
        if [[ "$val" != "" ]]; then
            PLATFORM_CP_DP_CORE_ASSIGN=$val
        fi
    fi

    if [[ "$PLATFORM_CP_DP_CORE_ASSIGN" = "" ]]; then
        platform_log_error "CP cores not set"
    else
        cp_cores=`echo $PLATFORM_CP_DP_CORE_ASSIGN | cut -f1 -d','`
    fi

    if [[ "$PLATFORM_CP_DP_CORE_ASSIGN" = "" ]]; then
        platform_log_error "DP cores not set"
    else
        dp_cores=`echo $PLATFORM_CP_DP_CORE_ASSIGN | cut -f2 -d','`
    fi
}

#
# Get the domain settings from the database. If set already, then ignore the
# values in the database.
#
function platform_hw_profile_uvf_get
{
    #
    # Get the number of vCPUS
    #
    val=`$PLATFORM_HW_PROFILE_TOOL -domain uvf -cpu`
    if [[ $? -ne 0 ]]; then
        $PLATFORM_HW_PROFILE_TOOL -domain uvf -cpu
    else
        if [[ "$val" != "" ]]; then
            PLATFORM_CP_DP_CORE_ASSIGN=$val
        fi
    fi

    if [[ "$PLATFORM_CP_DP_CORE_ASSIGN" = "" ]]; then
        platform_log_error "CP cores not set"
    else
        cp_cores=`echo $PLATFORM_CP_DP_CORE_ASSIGN | cut -f1 -d','`
    fi

    if [[ "$PLATFORM_CP_DP_CORE_ASSIGN" = "" ]]; then
        platform_log_error "DP cores not set"
    else
        dp_cores=`echo $PLATFORM_CP_DP_CORE_ASSIGN | cut -f2 -d','`
    fi
}

#
# Get the domain settings from the database. If set already, then ignore the
# values in the database.
#
function platform_hw_profile_lcp_get
{
    #
    # Get the VM memory
    #
    val=$($PLATFORM_HW_PROFILE_TOOL -domain uvf -mem)
    if [[ $? -ne 0 ]]; then
        $PLATFORM_HW_PROFILE_TOOL -domain uvf -mem
    else
        if [[ "$val" != "" ]]; then
            PLATFORM_LCP_VM_MEM_MB=$(echo | awk "{printf(\"%d\n\", \
                                                  $val * 1024)}")
        fi
    fi
}

function platform_hw_profile_get_dp_cores
{
    if [[ "$PLATFORM_CP_DP_CORE_ASSIGN" = "" ]]; then
        platform_hw_profile_get_settings
        if [[ "$PLATFORM_CP_DP_CORE_ASSIGN" = "" ]]; then
            platform_log_error "Failed to get PLATFORM_CP_DP_CORE_ASSIGN"
            echo ""
            return
        fi
    fi
    local dp_cores=`echo $PLATFORM_CP_DP_CORE_ASSIGN | cut -f2 -d','`
    if [[ $dp_cores =~ ^[0-9,-]+$ ]]; then
        echo $dp_cores
    else
        platform_log_error "Wrong dp_cores setting in PLATFORM_CP_DP_CORE_ASSIGN"
        echo ""
    fi
}

function platform_hw_profile_get_isolcpus_cfg
{
    # For RP in the distributed mode, we do not need isolcpus
    if is_distributed; then
        get_board_type
        if [[ $BOARDTYPE == "RP" ]]; then
            echo ""
            return
        fi
    fi
 
    local dp_cores=$(platform_hw_profile_get_dp_cores)
    local isolcpus_cfg=""
        
    if [[ $dp_cores != "" ]]; then
        isolcpus_cfg="isolcpus=${dp_cores}"
    fi
    echo $isolcpus_cfg
}

function platform_hw_profile_host_get
{
    #
    # Get the host memory
    #
    val=`$PLATFORM_HW_PROFILE_TOOL -hostmem`
    if [[ $? -ne 0 ]]; then
        $PLATFORM_HW_PROFILE_TOOL -hostmem
        return
    fi

    if [[ "$val" != "" ]]; then
        PLATFORM_HOST_MEM_MB=$(echo | awk "{printf(\"%d\n\", $val * 1024)}")
    fi

    #
    # Get the number of CPU cores
    #
    val=`$PLATFORM_HW_PROFILE_TOOL -hostncpu`
    if [[ $? -ne 0 ]]; then
        $PLATFORM_HW_PROFILE_TOOL -hostncpu
        return
    fi

    if [[ "$val" != "" ]]; then
        PLATFORM_HOST_NCPU=$val
    fi
}

function max_cpu_idx
{
    local cpu_list=$1

    if [[ "$cpu_list" != "" ]] ; then
        local fr=`echo $cpu_list | cut -d',' -f2- -s`
        local max_cpu=$(max_cpu_idx $fr)

        local f1=`echo $cpu_list | cut -d',' -f1`
        local min=`echo $f1 | cut -d'-' -f1`
        local max=`echo $f1 | cut -d'-' -f2 -s`
        if [[ "$max" == "" ]]; then
            max=$min
        fi
        (($max_cpu < $max)) && max_cpu=$max
        echo $max_cpu
    else
        echo 0
    fi
}

function platform_hw_profile_demand_get
{
    if [[ "$PLATFORM_CP_DP_CORE_ASSIGN" = "" ]]; then
        platform_hw_profile_get_settings
        if [[ "$PLATFORM_CP_DP_CORE_ASSIGN" = "" ]]; then
            platform_log_error "Failed to get PLATFORM_CP_DP_CORE_ASSIGN"
            return
        fi
    fi

    #
    # Get the total request memory from domains (sysadmin + xr + uvf)
    #
    PLATFORM_DOMAIN_REQUEST_MEM_MB=$(($PLATFORM_CALVADOS_VM_MEM_MB +\
                $PLATFORM_XR_VM_MEM_MB + $PLATFORM_LCP_VM_MEM_MB))

    #
    # count the total request CPU cores specified in $PLATFORM_CP_DP_CORE_ASSIGN
    #
    PLATFORM_DOMAIN_REQUEST_NCPU=$(( $(max_cpu_idx $dp_cores) + 1 ))
}

#
# Final fixup of values before we apply them to patch files
#
function platform_hw_profile_calvados_fixup
{
    #
    # Calvados wants the amount in Kb
    #
    if [[ "$PLATFORM_CALVADOS_VM_MEM_MB" != "" ]]; then
        PLATFORM_CALVADOS_VM_MEM_KB=$(echo | awk "{printf(\"%d\n\", $PLATFORM_CALVADOS_VM_MEM_MB * $MB_TO_KB_CONVERT)}")
    fi
}

#
# Retrieve the current hardware profile from GRUB
#
function platform_hw_profile_read_cmdline
{
    local rootdir=$1

    if [[ "$PROFILE" != "" ]]; then
        PLATFORM_HW_PROFILE=$PROFILE
        platform_log "platform hw profile from meta-data, ${PLATFORM_HW_PROFILE}"
    else 
        #
        # Find the linux cmdline info we need
        #
        find_cmdline $rootdir
        if [[ "$cmdline" = "" ]]; then
            return
        fi
        #
        # Check to see if a profile exists
        #
        grep -q "__hw_profile=" $cmdline
        if [[ $? -ne 0 ]]; then
            PLATFORM_HW_PROFILE=$PLATFORM_DEFAULT_HW_PROFILE
            return
        fi

        #
        # Extract the profile name
        #
        PLATFORM_HW_PROFILE=`sed 's/.*__hw_profile=\(\w*\).*/\1/g' $cmdline`
        if [[ "$PLATFORM_HW_PROFILE" = "" ]]; then
            PLATFORM_HW_PROFILE=$PLATFORM_DEFAULT_HW_PROFILE
            return
        fi
        platform_log "platform hw profile from cmd line, ${PLATFORM_HW_PROFILE}"
    fi

    #
    # Be a bit paranoid about the value passed in. Just in case someone feeds
    # us a random rc file like "../../myscript"
    #
    case $PLATFORM_HW_PROFILE in
    vrr) ;;
    vpe) ;;
    *)
        platform_log_error "Unknown platform hw profile $PLATFORM_HW_PROFILE"
        PLATFORM_HW_PROFILE=$PLATFORM_DEFAULT_HW_PROFILE
        return
    esac
}

#
# Called to override settings for calvados for a given hw profile
#
function platform_hw_profile_patch_calvados
{
    local rootdir=$1
    
    #
    # Nothing to do here as we need to dynamically adjust settings at 
    # running time.
    #
}

#
# Called to update bootstrap aux settings before calvados starts
#
function platform_hw_profile_update_sysadmin_aux_bootstrap
{
    local rootdir=$1
    local bootstrap_aux_file="${rootdir}/etc/rc.d/init.d/calvados_bootstrap_aux.cfg"

    platform_hw_profile_get_settings /
    if [[ "${PLATFORM_HW_PROFILE}" = "" ]]; then
        platform_log_error "PLATFORM_HW_PROFILE is unset"
        return 1
    fi

    #
    # Validate the profile before continuing
    #
    platform_log "sysadmin start hook for hw profile, ${PLATFORM_HW_PROFILE}"

    if [[ ("${PLATFORM_XR_VM_MEM_MB}" = "") || (${PLATFORM_XR_VM_MEM_MB} -lt 1) ]]; then
        platform_log_error "PLATFORM_XR_VM_MEM_MB is not properly set"
        return 1
    fi

    local xr_vm_mem_kb=$((PLATFORM_XR_VM_MEM_MB * MB_TO_KB_CONVERT))
    
    local lcp_vm_mem_kb=0
    if ! is_distributed; then
        if [[ ("${PLATFORM_LCP_VM_MEM_MB}" = "") \
              || (${PLATFORM_LCP_VM_MEM_MB} -lt 1) ]]; then
            platform_log_error "PLATFORM_LCP_VM_MEM_MB is not properly set"
            return 1
        fi
        lcp_vm_mem_kb=$((PLATFORM_LCP_VM_MEM_MB  * MB_TO_KB_CONVERT))
    fi
    
    
    for pattern  in "s/VIRT_XR_VM_MEM=.*/VIRT_XR_VM_MEM=${xr_vm_mem_kb}/g"   \
                    "s/VIRT_LCP_VM_MEM.*/VIRT_LCP_VM_MEM=${lcp_vm_mem_kb}/g" 
    do
        sed -i ${pattern} ${bootstrap_aux_file}
        if [[ $? -ne 0 ]]; then
            platform_log_error "Failed to ${pattern} for ${bootstrap_aux_file}"
            return 1
        fi
    done

    platform_log "Completed update ${bootstrap_aux_file} for hw profile ${PLATFORM_HW_PROFILE}"
    return 0
}

function reset_profile_file
{
    local profile=$1
    local cpu_mem=$2
    local profile_rc=`find_file ${profile}.profile`

    case $cpu_mem in
    "cpu")
        sed -i "s/.*\(DOMAIN_CPU_INFO=\).*/\t# \1/g" $profile_rc
        ;;
    "mem")
        sed -i "s/.*\(DOMAIN_MEM_INFO=\).*/\t# \1/g" $profile_rc
        ;;
    *)
        platform_log "Invalid parameter $cpu_mem for reset_profile_file"
        ;;
    esac
}

#
# check system total resources (cpu/memory) against requested resources from each domain
# if not match, reset to use default configuration
#
function platform_hw_profile_adjust_config
{
    local reboot_required=FALSE

    find_cmdline

    #
    # get total of requested cpu/memory 
    #
    platform_hw_profile_demand_get

    open_grub_cmdline
    local grub_file="$TMPMNT/boot/grub/menu.lst"

    if (( $PLATFORM_HOST_NCPU != $PLATFORM_DOMAIN_REQUEST_NCPU )); then
        platform_log_console "Total available CPU cores($PLATFORM_HOST_NCPU) \
mismatch total request CPU cores($PLATFORM_DOMAIN_REQUEST_NCPU)\
, reset CP/DP CPU assignment to default for hw profile, $PLATFORM_HW_PROFILE"

        reset_profile_file  $PLATFORM_HW_PROFILE cpu

        grep -q "__hw_profile_cpu=" $cmdline 
        if [[ $? -eq 0 ]]; then
            set_cmdline $grub_file __hw_profile_cpu

            platform_hw_profile_default_uvf_get $PLATFORM_HOST_NCPU
            set_cmdline $grub_file isolcpus $dp_cores
            platform_log_console "Automatically adjust grub isolcpus setting to $isolcpus and reboot"
            reboot_required=TRUE
        fi
    fi

    if (( $PLATFORM_HOST_MEM_MB != $PLATFORM_DOMAIN_REQUEST_MEM_MB)); then
        platform_log_console "Total available memory(${PLATFORM_HOST_MEM_MB}MB) \
mismatch total request memory(${PLATFORM_DOMAIN_REQUEST_MEM_MB}MB)\
, reset memory configuration to default for hw profile, $PLATFORM_HW_PROFILE"

        reset_profile_file  $PLATFORM_HW_PROFILE mem

        grep -q "__hw_profile_vm_mem_gb=" $cmdline 
        if [[ $? -eq 0 ]]; then
            set_cmdline $grub_file __hw_profile_vm_mem_gb
            reboot_required=TRUE
        fi

    fi

    platform_log_exec cat $grub_file
    close_grub_cmdline

    if [[ "$reboot_required" = "TRUE" ]]; then
        /sbin/reboot -f
    else
        platform_hw_profile_get_settings
    fi
}

#
# Called to override settings for the host for a given hw profile
#
function platform_hw_profile_patch_hostos
{
    local rootdir=$1

    platform_hw_profile_get_settings $rootdir

    if [[ "$PLATFORM_HW_PROFILE" = "" ]]; then
        return
    fi

    #
    # Useful point to print the hardware profile information
    #
    platform_hw_profile_calvados_info
    platform_hw_profile_xr_info
    platform_hw_profile_uvf_info
    platform_hw_profile_tpa_info

    platform_log "Patch hostos for hw profile, $PLATFORM_HW_PROFILE"

    #
    # If an override function is defined, call it
    #
    declare -F platform_hw_profile_patch_host_override &>/dev/null && \
                     platform_hw_profile_patch_host_override $rootdir

    #
    # Now fix up calvados for the values returned that we allow to be modified
    #
    file=${rootdir}/etc/init.d/calvados_bootstrap.cfg

    if [[ "$PLATFORM_CALVADOS_VM_CPU" != "" ]]; then
        #
        # Treat 0 as just use the default, 1. This allows us to share cores.
        #
        if [[ "$PLATFORM_CALVADOS_VM_CPU" != "0" ]]; then
            sed --in-place \
                "s/VIRT_RP_SMP=.*/VIRT_RP_SMP=$PLATFORM_CALVADOS_VM_CPU/g" \
                $file
        fi
    fi

    if [[ "$PLATFORM_CALVADOS_VM_MEM_MB" != "" ]]; then
        sed --in-place \
            "s/VIRT_RP_MEM=.*/VIRT_RP_MEM=$PLATFORM_CALVADOS_VM_MEM_KB/g" \
            $file
    fi

    platform_log "Default calvados config:"
    platform_log_exec cat $file

    platform_log "Completed patch for hw profile, $PLATFORM_HW_PROFILE"
}

#
# Set the affinity of a process to a range of cores "a-b"
#
process_set_affinity()
{
    local pid=$1
    local pidname=$2
    local target_cores=$3

    if [[ "$pidname" = "" ]]; then
        pidname=`ps -p $pid -ocomm= 2>/dev/null`
        if [[ "$pidname" = "" ]]; then
            return
        fi
    fi

    local current_cores=`taskset -cp $pid | cut -d':' -f2 | sed 's/ //g'`
    if [[ "$current_cores" = "$target_cores" ]]; then
        #
        # No change
        #
        continue
    fi

    taskset -a -cp $target_cores $pid >& /dev/null
    if [[ $? -ne 0 ]]; then
        # Check if this process still exists
        pidname=$(ps -p $pid -o comm= 2>/dev/null)
        #
        # Logging is suppressed on ThinXR to hide warnings about kernel
        # processes.
        #
        if [[ "$pidname" != "" ]] && ! am_i_thinxr; then
            platform_log_error "Failed to set CPU affinity for \"$pidname\", pid $pid to $target_cores"
        fi
    else
        platform_log "Set CPU affinity for \"$pidname\", pid $pid to $target_cores"
    fi
}

#
# Debug function to dump the cgroup membership
#
# e.g.:
#
# Wed Mar  1 14:07:41 UTC 2017 (-bash): pidname                 pid      cores cgroup
# Wed Mar  1 14:07:41 UTC 2017 (-bash): =======                 ===    ======= ======
# Wed Mar  1 14:07:41 UTC 2017 (-bash): [init]                    1        0,1 default-sdr--2.libvirt-lxc
# Wed Mar  1 14:07:41 UTC 2017 (-bash): [oom.sh]               1676        0,1 uvfcp
# Wed Mar  1 14:07:41 UTC 2017 (-bash): [cgroup_oom.sh]        1692        0,1 uvfcp
# Wed Mar  1 14:07:41 UTC 2017 (-bash): [oom.sh]               1693        0,1 uvfcp
# Wed Mar  1 14:07:41 UTC 2017 (-bash): [cgroup_oom]           1710        0,1 uvfcp
# Wed Mar  1 14:07:41 UTC 2017 (-bash): [app_config_back]      2158        0,1 uvfcp
# Wed Mar  1 14:07:41 UTC 2017 (-bash): [dbus-daemon]          2182        0,1 uvfcp
# Wed Mar  1 14:07:41 UTC 2017 (-bash): [sshd]                 2201        0,1 uvfcp
# Wed Mar  1 14:07:41 UTC 2017 (-bash): [rpcbind]              2208        0,1 uvfcp
# Wed Mar  1 14:07:41 UTC 2017 (-bash): [syslogd]              2304        0,1 uvfcp
# Wed Mar  1 14:07:41 UTC 2017 (-bash): [xinetd]               2319        0,1 uvfcp
# Wed Mar  1 14:07:41 UTC 2017 (-bash): [crond]                2334        0,1 uvfcp
# Wed Mar  1 14:07:41 UTC 2017 (-bash): [sh]                   2591        0,1 uvfcp
# Wed Mar  1 14:07:42 UTC 2017 (-bash): [ds_startup.sh]        3313        0,1 uvfcp
#
function dump_process_cgroup_info ()
{
    local format="%-20s %6d %10s %s"
    local formats="%-20s %6s %10s %s"

    platform_log "$(printf "$formats" "pidname" "pid" "  cores" "cgroup")"
    platform_log "$(printf "$formats" "=======" "===" "=======" "======")"

    for pid in $(ps -o pid= -e)
    do
        local pidname=$(ps -p $pid -o comm=)
        if [[ "$pidname" = "" ]]; then
            continue
        fi

        local cgroup=$(grep cpuset /proc/$pid/cgroup | sed 's/.*\///g')
        local cores=$(taskset -cp $pid | cut -d':' -f2 | sed 's/ //g')

        platform_log "$(printf "$format" "[$pidname]" $pid $cores $cgroup)"
    done
}

function set_cpusets
{
    echo $cp_cores > /dev/cgroup/cpuset/machine/sysadmin.libvirt-lxc/cpuset.cpus
    echo $cp_cores > /dev/cgroup/cpuset/machine/default-sdr--1.libvirt-lxc/cpuset.cpus
}

#
# Make sure any newborn processes move to the correct cores
#
function move_processes_to_child_cgroups_repeat 
{
    if [[ "$cp_cores" = "" ]]; then
        platform_hw_profile_get_settings
        if [[ "$cp_cores" = "" ]]; then
            platform_log_error "Failed to get CP cores"
        fi
    fi

    while true; do
        #
        # Try to move processes to top_spirit; do not force processes
        # that are already in a cgroup.
        #
        move_processes_to_child_cgroups top_spirit ""

        #wake up periodically and do task move if needed
        sleep 300
    done
}

#
# Make sure any newborn processes move to the correct core
#
function move_processes_to_child_cgroups_repeat_uvf
{        
    ( 
        while true; do
            #
            # Force move into ufcp even if in another cgroup.
            # Sleep first so that the ucode has time to start the first time.
            #
            sleep 60
            move_processes_to_child_cgroups uvfcp force
        done
    ) &
}

#
# Move found processes to either the control or dataplane child cgroups
#
function move_processes_to_child_cgroups ()
{
    local default_cgroup=$1
    local force_move=$2

    local cgroup_prefix=/dev/cgroup
    local exclude_pid_list=
    local something_changed=

    #
    # This loop is a bit CPU intensive, so only do it if the system is 
    # not heavily loaded
    #
    # we do this once every 5 minutes

    #
    # First time init
    #
    if [[ "$move_processes_to_child_cgroups_init" = "" ]]; then
        move_processes_to_child_cgroups="done"
        pid_failed_to_move_kthreadd=1
        pid_failed_to_move_libvirtd=1
    fi

    if [[ "$cp_cores" = "" ]]; then
        platform_hw_profile_get_settings
        if [[ "$cp_cores" = "" ]]; then
            platform_log_error "CP cores cannot be set"
            return
        fi
    fi

    if [[ "$default_cgroup" = "uvfcp" ]]; then

        # No need to move /root/vpe, start_vpe.sh processes
        for pid in $(pidof /root/vpe); do
            exclude_pid_list[$pid]=1
        done
        for pid in $(pgrep start_vpe.sh); do
            exclude_pid_list[$pid]=1
        done

        #
        # It is safer to look at all processes as if an earlier pid had
        # been associated with the dp and respawned as a pid that was already
        # associated with the cp then it would not appear in the parent task
        # list and would be in the wrong group
        #
        # for pid in $(cat `$cgroup_prefix/cpuset/tasks)
        pids=$(ps -o pid= -e)
    else
        #
        # Only processes trying to sit in the dp cores
        #
        # pids=$(ps -e -o psr,pid | awk '{if($1>=$cores)print $2;}')

        #
        # Only processes not in a cgroup
        #
        pids=$(cat $cgroup_prefix/cpuset/tasks)
    fi

    for pid in $pids
    do
        if [[ ! -f /proc/$pid/cgroup ]]; then
            continue
        fi

        #Skip moving the init in uvf to the uvfcp cgroup
        if [[  $default_cgroup = "uvfcp"  &&  $pid = 1  ]]; then
               continue   
        fi

        #
        # Check it is not a transient process.
        #
        local pidname=$(ps -p $pid -o comm=)
        if [[ "$pidname" = "" ]]; then
            continue
        fi

        #
        # Bash 3 does not have associatie arrays and interprets a process name 
        # like "kernel/3" as divide by 3... Take out characters that might cause 
        # issues.
        #
        # Also would be a great way to hack the system by creating a carefully
        # crafted process name
        #
        pidname=`echo $pidname | tr -c 'A-Za-z0-9' '_'`

        #
        # If we have tried to move this process before and it failed, do 
        # not repeat
        #
        # Die to lack of associative arrays in bash3 I need to do this nonsense.
        #
        local varname=pid_failed_to_move_$pidname
        if [[ ${!varname} != "" ]]; then
            continue
        fi

        #
        # Map the proess to a cgroup.
        #
        local desired_cores=
        local desired_cgroup=
        local current_cgroup=$(grep cpuset /proc/$pid/cgroup | sed 's/.*\///g')

        if [[ ${exclude_pid_list[$pid]} = 1 ]]; then
            # no need to do anything for those pid in the exclude_pid_list
            continue
        else
            #
            # Is not in a cgroup, try to move to the default
            #
            desired_cgroup=$default_cgroup
            desired_cores=$cp_cores
        fi

        if [[ "$current_cgroup" = $desired_cgroup ]]; then
            #
            # Is already in the desired cgroup
            #
            continue
        fi

        #
        # If already in a cgroup, then only move if force is on
        #
        if [[ "$current_cgroup" != "" && $force_move = "" ]]; then
            continue
        fi

        local current_cores=$(taskset -cp $pid | cut -d':' -f2 | sed 's/ //g')

        process_set_affinity $pid "$pidname" $desired_cores

        (echo $pid > $cgroup_prefix/cpuacct/$desired_cgroup/tasks) 2>/dev/null

        #
        # Check if we are blocked from moving this PID. Could be a kernel 
        # thread.
        #
        if [[ $? -ne 0 ]]; then
            #
            # Need to use readonly to trick bash into creating a global var
            #
            readonly pid_failed_to_move_$pidname=1
            
            #
            # Logging is suppressed on ThinXR to hide warnings about kernel
            # processes.
            #
            if ! am_i_thinxr; then
                platform_log "Move failed for \"$pidname\", pid $pid, taskset $current_cores, cgroup \"$current_cgroup\" to cgroup \"$desired_cgroup\""
            fi
            continue
        fi

        echo $pid > $cgroup_prefix/cpuset/$desired_cgroup/tasks
        echo $pid > $cgroup_prefix/cpu/$desired_cgroup/tasks

        platform_log "Move success for \"$pidname\", pid $pid, taskset $current_cores, cgroup \"$current_cgroup\" to cgroup \"$desired_cgroup\""

        something_changed=1

        #
        # Make sure the CP affinity is set correctly. It should be.
        #
        if [[ "$current_cores" = "$desired_cores" ]]; then
            #
            # No change
            #
            continue
        fi

    done

    if [[ "$something_changed" != "" ]]; then
        dump_process_cgroup_info
    fi
}

#This is a variation of the above function
function move_procto_topnode ()
{
    local runcount=$1

    local CGROUP_MOUNT_LOG_FILE=/tmp/cgroupmount$$.log
    local MOVE_LOG="/tmp/cgroup_move_error"

    #
    # Create the exclusion list - tasks to be retained in root cgroup
    #
    for pid in `ps --ppid $KTHREAD_PID -o pid | grep -v PID`
    do
	exclude_list[$pid]=1
    done

    #
    # Add any other tasks in the exclude_list here -- 
    #
    exclude_list[$KTHREAD_PID]=1
    for pid in `ps -efL | egrep "(telnet|libvirtd)" | grep -v grep | awk '{print $4}'`
    do
	exclude_list[$pid]=1
    done

    #
    # Populate the include list file.
    #
    rm /tmp/tasks_include$$ &> /dev/null
    for pid in `cat $cgroup_mount_pt/cpu/tasks`
    do
	if [[ ${exclude_list[$pid]} -ne 1 ]]; then
	    echo $pid >> /tmp/tasks_include$$
	fi
    done

    echo "Tasks in inclusion list:" >> /tmp/tasks_includename$$
    for pid in `cat /tmp/tasks_include$$`
    do
	ps -p $pid -o comm= >> /tmp/tasks_includename$$
    done

    platform_log "Process move iteration $runcount"
    platform_log_exec cat /tmp/tasks_includename$$ 

    rm ${MOVE_LOG} &> /dev/null
    echo "Move failed for following:" >> $CGROUP_MOUNT_LOG_FILE
    echo "==========================" >> $CGROUP_MOUNT_LOG_FILE
    for pid in `cat /tmp/tasks_include$$`
    do
	echo $pid > $cgroup_mount_pt/cpu/top_spirit/tasks 2> ${MOVE_LOG}
	echo $pid > $cgroup_mount_pt/cpuset/top_spirit/tasks 2> ${MOVE_LOG}
	echo $pid > $cgroup_mount_pt/cpuacct/top_spirit/tasks 2> ${MOVE_LOG}
	echo $pid > $cgroup_mount_pt/memory/top_spirit/tasks 2> ${MOVE_LOG}
	if [[ $? -ne 0 ]]; then
	    MOVE_ERROR=`cat ${MOVE_LOG}`
	    echo "Failed to move PID=\"${pid}\" due to: \"${MOVE_ERROR}\"." >> ${CGROUP_MOUNT_LOG_FILE}
      if [[ $? != 0 ]];then
	   progname=`ps -p $pid -o comm=`
	   echo "failed to move pid $pid $progname"
      fi
	    rm ${MOVE_LOG}
	fi
    done
    echo "==========================" >> $CGROUP_MOUNT_LOG_FILE

    for pid in `cat $cgroup_mount_pt/cpu/tasks`; do
        echo "pid $pid `ps -p $pid -ocomm=`" >> $CGROUP_MOUNT_LOG_FILE
    done
    echo "========================" >> $CGROUP_MOUNT_LOG_FILE

    #echo "Tasks in the top_spirit group " >> $CGROUP_MOUNT_LOG_FILE
    #echo "===========================" >> $CGROUP_MOUNT_LOG_FILE
    #cat $cgroup_mount_pt/cpu/top_spirit/tasks >> $CGROUP_MOUNT_LOG_FILE
    for pid in `cat $cgroup_mount_pt/cpu/top_spirit/tasks`; do
        echo "pid $pid `ps -p $pid -ocomm=`" >> $CGROUP_MOUNT_LOG_FILE
    done
    platform_log_exec cat $CGROUP_MOUNT_LOG_FILE

    #
    # Cleanup.
    #
    rm -f /tmp/tasks_include$$
    rm -f /tmp/tasks_includename$$
    rm ${MOVE_LOG} &> /dev/null
    rm ${CGROUP_MOUNT_LOG_FILE} &> /dev/null

    dump_process_cgroup_info
}

#
# Get the default settings for all domains.
#
function platform_hw_profile_get_settings
{
    source_hw_profiles

    local rootdir=$1

    #
    # Read the hw profile if set
    #
    platform_hw_profile_read_cmdline

    #
    # Read the database to learn hw profile requirements
    #
    platform_hw_profile_init_get
    platform_hw_profile_calvados_get
    platform_hw_profile_xr_get
    platform_hw_profile_uvf_get
    platform_hw_profile_lcp_get

    #
    # get host total cpu cores and memory size
    #
    platform_hw_profile_host_get

    #
    # Finalize values for patching to use
    #
    platform_hw_profile_calvados_fixup
}

#
# tpa_metadata_get_ncpus returns number of TPA CPU cores
# defined in meta-data
#
tpa_metadata_get_ncpus()
{
    tpa_cores_min=`echo $APP_HOST_CPUS | cut -f1 -d'-'`
    tpa_cores_max=`echo $APP_HOST_CPUS | cut -f2 -d'-'`
    tpa_cores=$(( tpa_cores_max - tpa_cores_min + 1 ))
    echo $tpa_cores
}

#
# tpa_can_modify_host_ncpus check if TPA meta-data asks for more
# CPUs than the default settings.
# 
tpa_can_modify_host_ncpus()
{
    local HOST_NCPUS=`cat /proc/cpuinfo | grep processor | wc -l`

    if [[ "$APP_HOST_CPUS" != "" ]]; then
        if [[ $HOST_NCPUS -gt $(tpa_metadata_get_ncpus) ]]; then
            true
            return                
        fi
    fi
    
    false
    return
}

#
# tpa_can_modify_numa_nodes
# Check if we have $APP_HOST_CPUSET_MEMS defined in YAML file
# so we can overwrite CGROUP default settings for CPU NUMA nodes.
tpa_can_modify_numa_nodes()
{
    if [[ "$APP_HOST_CPUSET_MEMS" != "" ]]; then
        true
        return
    fi

    false
    return
}

#
# tpa_can_modify_host_memory check if TPA meta-data asks for more
# Memory than the default settings.
# 
tpa_can_modify_host_memory()
{
    HOST_MEM=`cat /proc/meminfo | grep MemTotal | awk '{print int($2 / (1024*1024))}'`

    if [[ "$APP_HOST_MEM_GB" != "" ]]; then
        app_host_mem_mb=$(echo | awk "{printf(\"%d\n\", $APP_HOST_MEM_GB * 1024)}")
        host_mem_mb=`cat /proc/meminfo | grep MemTotal | awk '{print int($2 / 1024)}'`
        if [[ $host_mem_mb -gt $app_host_mem_mb ]]; then    
            true
            return
        fi
    fi

    false
    return
}

