Step1: Build SBC in Azure

Below Diagram Depicts the High Level Architecture of SWe deployment in Azure.

Link of some important URL to get the latest documentation

https://support.sonus.net/display/SBXDOC91/Instantiating+SBC+SWe+in+Azure

A High Availability Front End (HFE) is deployed for each pkt interface.

Below Diagram depicts a Service Provider Multi-Tenant Architecture with Teams Direct Routing in a Derived Domain model.

In Order to build the VMs in the Azure, created a below Low Level Architecture diagram:

Here s a quick shell script to build the VMs in one shot:

Create the Basic Infra Components:

1. Upload the HFE Configuration shell script below with a file name : HFE_AZ.sh

#!/bin/bash

#############################################################
#
# Copyright (c) 2019 Ribbon Communications Inc.
#
# All Rights Reserved.
# Confidential and Proprietary.
#
# HFE_AZ.sh
#
# Module Description:
# Script to enable HFE(High-Availability Front End) instance as frontend for 
# PKT ports of SBC.
# Call "HFE_AZ.sh setup" - to configure HFE with Secondary or Primary IPs
# The script will perform the following steps when called from cloud-init (setup function):
# 1) Save and clear old iptables rules : preConfigure
# 2) Enalbe IP Forwarding :configureNATRules
# 3) Read variables from natVars.input configured from userdata: readConfig
# 4) Determine interface names and GW IPs on HFE: setInterfaceMap
# 5) Extract the interface IPs required to configure the HFE for pkt0/pkt1
#    ports: extractHfeConfigIps
# 6) Setup DNATs, SNATs and routes to route traffic through HFE to/from SBC PKT ports: configureNATRules
# 7) Configure Management interface to HFE on ens6: configureMgmtNAT
# 8) Log applied iptables configuration and routes: showCurrentConfig 
# 9) Check connectivity to currently Active SBC using ping and switchover to Standby SBC
#    if connection is lost to current Active: checkSbcConnectivity
#
#
# Call "HFE_AZ.sh cleanup" to clear all IP tables and Routes on the HFE.
# 1) Save and clear old iptables rules : preConfigure
# 2) Read variables from natVars.input configured from userdata: readConfig
# 3) Determine interface names and GW IPs on HFE: setInterfaceMap
# 4) Extract the interface IPs required to configure the HFE for pkt0/pkt1
#    ports: extractHfeConfigIps
# 5) Route clean up for Pkt0/Pkt1 subnet and remote machine's public IP: routeCleanUp
#
# This option is useful to debug connectivity of end-point with HFE, after
# calling this no packet is forwarded to SBC, user can ping IP on ens4 to
# make sure connectivity between end-point and HFE is working fine.
# Once this is done user MUST reboot HFE node to restore all forwarding rules
# and routes.
# 
#
# NOTE: This script is run by cloud-init or systemd in HFE instance.
#
# This script should be uploaded to a container in an Azure Storage Account
# HFE Node user data then downloads and runs with cloud-init.
# See Ribbon Documentation for more information.
#
#
#############################################################

declare -A interfaceMap
declare -A commandMap

HFERoot="/opt/HFE"
varFile="$HFERoot/natVars.input"
confLogFile="$HFERoot/log/HFE_conf.log"
statusLogFile="$HFERoot/log/HFE.log"
oldRules="$HFERoot/iptables.rules.prev"
hfeLogRotateConf="/etc/logrotate.d/hfe"
APP_INSTALL_MARKER="$HFERoot/app_install_marker"

IPTABLES="$(command -v iptables)"
CURL="$(command -v curl)"

ACTIVE_SBC_IP_ARR=""
STANDBY_SBC_IP_ARR=""
HFE_ETH0_IP_ARR=""
ACTIVE_SBC_IP_PKT1_ARR=""
STANDBY_SBC_IP_PKT1_ARR=""
HFE_ETH1_IP_ARR=""
MODE=""
gRetVal=""
readonly INTF_TYPE_MASTER=2
readonly INTF_TYPE_SLAVE=1
readonly INTF_TYPE_NORMAL=0
readonly MAX_QUEUE_LENGTH=8192


PRIMARY_IP=0

RG_NAME=""    # Resource Group Name
SUB_ID=""     # Subscription ID

# REST VARS 
apiversion="api-version=2018-10-01"
resource="resource=https://management.azure.com/"

DEBUG_MODE="0"
INIT_LOG=1

PROG=${0##*/}

usage()
{
    echo $1
    echo "usage: $PROG <setup | cleanup>"
    echo "Example:"
    echo "$PROG setup - Setup HFE with Secondary IPs" 
    echo "$PROG cleanup - Cleanup IP tables and route on HFE" 
    exit
}

timestamp()
{
 date +"%Y-%m-%d %T"
}

timestampDebug()
{
 date +"%F %T,%3N"
}

loggerHfe()
{
    echo $(timestamp) "$1" >> $confLogFile 2>&1
}

statusLoggerHfeDebug()
{
    if [ "$DEBUG_MODE" -eq "1" ]; then
        echo $(timestamp) "$1" >> $statusLogFile 2>&1
    fi
}

statusLoggerHfe()
{
    echo $(timestamp) "$1" >> $statusLogFile 2>&1
}

doneMessage()
{
    loggerHfe " =========================    DONE HFE_AZ.sh     =========================================="
    exit 
}

errorAndExit()
{
    loggerHfe " Error: $1"
    doneMessage
}


clearOldIptableRules()
{
    # reset the default policies in the filter table.
    $IPTABLES -P INPUT ACCEPT
    $IPTABLES -P FORWARD ACCEPT
    $IPTABLES -P OUTPUT ACCEPT

    # reset the default policies in the nat table.
    $IPTABLES -t nat -P PREROUTING ACCEPT
    $IPTABLES -t nat -P POSTROUTING ACCEPT
    $IPTABLES -t nat -P OUTPUT ACCEPT

    # reset the default policies in the mangle table.
    $IPTABLES -t mangle -P PREROUTING ACCEPT
    $IPTABLES -t mangle -P POSTROUTING ACCEPT
    $IPTABLES -t mangle -P INPUT ACCEPT
    $IPTABLES -t mangle -P OUTPUT ACCEPT
    $IPTABLES -t mangle -P FORWARD ACCEPT

    # flush all the rules in the filter and nat tables.
    $IPTABLES -F
    $IPTABLES -t nat -F
    $IPTABLES -t mangle -F

    # erase all chains that's not default in filter and nat table.
    $IPTABLES -X
    $IPTABLES -t nat -X
    $IPTABLES -t mangle -X
}

getSubnetPrefix()
{
    local SBC_NAME=$1
    if [[ $2 == "PKT0" ]]; then
        IF="2"
    else
        IF="3"
    fi

    vmInstance=$($CURL "https://management.azure.com/subscriptions/$SUB_ID/resourceGroups/$RG_NAME/providers/Microsoft.Compute/virtualMachines/$SBC_NAME?$apiversion" -H "$authhead")
    networkInterfaceId=$(echo $vmInstance | $JQ .properties.networkProfile.networkInterfaces[$IF].id | sed 's/"//g')
    subnetId=$($CURL "https://management.azure.com$networkInterfaceId?$apiversion" -H "$authhead" | $JQ .properties.ipConfigurations[0].properties.subnet.id | sed 's/"//g')
    subnetPrefix=$($CURL "https://management.azure.com$subnetId?$apiversion" -H "$authhead" | $JQ .properties.addressPrefix | sed 's/"//g')
    loggerHfe " Got Subnet Prefix for $2 - $subnetPrefix"
    echo $subnetPrefix
}

#$1 = Active SBC NAME
#$2 = PKT port
#$3 = Gateway
#$4 = Interface
configureSubnetRoute()
{
    subnetPrefix=$(getSubnetPrefix $1 $2)
    declare -g subnetPrefix_$2="$subnetPrefix"
    statusLoggerHfeDebug "set subnetPrefix_$2 as $subnetPrefix"
    $IP route | grep -E "$subnetPrefix.*$4"
    if [ $? -eq 0 ]; then
        loggerHfe " Route already availabe to $2 subnet"
    else
        $IP route add $subnetPrefix via $3 dev $4
        if [ $? -eq 0 ]; then
            loggerHfe " Set route to reach $2 subnet"
        else
            errorAndExit "Failed to set route to $subnetPrefix"
        fi
    fi
}

# $1 - IP / Subnet Prefix
# $2 - Gateway
# $3 - Interface
removeRoute()
{
    $IP route | grep -E "$1.*$2.*$3"
    if [ $? -eq 0 ]; then
        $IP route del $1 via $2 dev $3
        if [ $? -eq 0 ]; then
            loggerHfe " Route deleted to for $1 "
        else
            errorAndExit " Failed to delete route for $1 "
        fi
    else
        loggerHfe " Route not available for $1 "
    fi
}
# $1 - IP / Subnet Prefix
# $2 - Gateway
# $3 - Interface
verifyRoute()
{
    statusLoggerHfeDebug "verifyRoute $1 $2 $3"
    $IP route | grep -E "$1.*$3"
    if [ $? -ne 0 ]; then
        statusLoggerHfeDebug "Adding missing route for $1."
        $IP route add $1 via $2 dev $3
    fi
}

verifyRoutes()
{
    if [[ $MODE == "single" ]]; then
        verifyRoute $subnetPrefix_PKT0 $ETH3_GW $ETH3
        verifyRoute $subnetPrefix_PKT1 $ETH4_GW $ETH4
    else
        var="subnetPrefix_$SBC_PKT_PORT_NAME"
        verifyRoute "${!var}" $ETH2_GW $ETH2
    fi

    IFS=',' read -ra SSH_IP_LIST <<< "$REMOTE_SSH_MACHINE_IP"
    for REMOTE_MACHINE_IP in "${SSH_IP_LIST[@]}"; do
       verifyRoute $REMOTE_MACHINE_IP $MGMT_GW $MGMT
    done
}

routeCleanUp()
{
    loggerHfe " Route clean up for Pkt0 and Pkt1 IP and remote machine's public IP."

    if [[ $MODE == 'single' ]]; then 
        #PKT0
        pkt0SubnetPrefix=$(getSubnetPrefix $ACTIVE_SBC_VM_NAME "PKT0")
        removeRoute $pkt1SubnetPrefix $ETH3_GW $ETH3
        #PKT1
        pkt1SubnetPrefix=$(getSubnetPrefix $ACTIVE_SBC_VM_NAME "PKT1")
        removeRoute $pkt1SubnetPrefix $ETH4_GW $ETH4
    else
        subnetPrefix=$(getSubnetPrefix $ACTIVE_SBC_VM_NAME $SBC_PKT_PORT_NAME)
        removeRoute $subnetPrefix $ETH2_GW $ETH2
    fi

    ## Cleanup Ip route from Remote Machine to HFE
    IFS=',' read -ra SSH_IP_LIST <<< "$REMOTE_SSH_MACHINE_IP"
    for REMOTE_MACHINE_IP in "${SSH_IP_LIST[@]}"; do
       removeRoute $REMOTE_MACHINE_IP $MGMT_GW $MGMT
    done
}

verifyPackages()
{
    local missingCmd=0

    commandMap["iptables"]=$IPTABLES
    commandMap["curl"]=$CURL
    commandMap["jq"]=$JQ
    commandMap["conntrack"]=$CONNTRACK
    commandMap["fping"]=$FPING
    commandMap["ip"]=$IP

    for i in "${!commandMap[@]}"
    do
        if [[ ${commandMap[$i]} == "" ]]; then
            loggerHfe "Required packages $i is missing."
            missingCmd=1
        fi
    done

    if [ $missingCmd -eq 1 ]; then
       errorAndExit "Missing required packages. Exiting!!!"
    fi

}

prepareHFEInstance()
{
    ### Configure ip_forward - Enable ip forwarding
    echo 1 >  /proc/sys/net/ipv4/ip_forward

    if [ ! -f $APP_INSTALL_MARKER ]; then
        ### Install required packages

        if [[ $(command -v yum) == "" ]]; then
            sudo apt-get update
            sudo apt-get -y install jq
            sudo apt-get -y install conntrack
            sudo apt-get -y install fping
        else
            sudo yum -y update
            sudo yum -y install epel-release
            if [ $? -ne 0 ]; then
                majorOsVer=$(cat /etc/os-release | grep VERSION_ID | awk -F= '{print $2}' | sed 's/"//g' | sed 's/\..*//')
                sudo yum -y install wget
                cd /tmp/
                wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-$majorOsVer.noarch.rpm
                epelRpm=$(ls epel*.rpm)
                sudo yum -y install $epelRpm
            fi
            sudo yum -y install jq
            sudo yum -y install conntrack
            sudo yum -y install fping
        fi
        setLogRotate
    fi

    JQ="$(command -v jq)"
    CONNTRACK="$(command -v conntrack)"
    FPING="$(command -v fping)"
    IP="$(command -v ip)"
    verifyPackages

    # If we get this far, packages are good
    echo 1 > $APP_INSTALL_MARKER
}

preConfigure()
{
    if [ -f $confLogFile ]; then
        mv -f $confLogFile $confLogFile.prev
    fi

    ### Redirect all echo $(timestamp) to file after writing ip_forward
    loggerHfe " ==========================   Starting HFE_AZ.sh  ============================"
    loggerHfe " Enabled IP forwarding"
    loggerHfe " This script will setup DNAT, SNAT and IP forwarding."
    loggerHfe " Save old rules in $HFERoot/firewall.rules"

    ### Save old iptable rules
    sudo iptables-save > $oldRules
    if [ "$?" = "0" ]; then
        ### Clear the iptables
        clearOldIptableRules
    else
        errorAndExit "Cound not save old iptables rules. Exiting"
    fi
}

getGatewayIp()
{
    local interface=$1
    cidrIp=$($IP route | grep "$interface proto kernel scope link" | awk -F " " '{print $1}' | awk -F "/" '{print $1}')
    finalOct=$(echo $cidrIp | awk -F "." '{print $4}')
    gwOctet=$(( finalOct + 1 ))
    gwIp=$(echo $cidrIp | awk -v var="$gwOctet" -F. '{$NF=var}1' OFS=.)
    echo $gwIp
}

setInterfaceMap()
{
    interfaceIds=$($CURL "https://management.azure.com/subscriptions/$SUB_ID/resourceGroups/$RG_NAME/providers/Microsoft.Compute/virtualMachines/$HFE_NAME?$apiversion" -H "$authhead" | $JQ .properties.networkProfile.networkInterfaces)
    interfaceNum=$(echo $interfaceIds | $JQ length)

    for i in $(seq 0 $((interfaceNum-1))); do
        interfaceId=$(echo $interfaceIds | $JQ .[$i].id | sed 's/"//g')
        privIp=$($CURL "https://management.azure.com$interfaceId?$apiversion" -H "$authhead" | $JQ .properties.ipConfigurations[0].properties.privateIPAddress | sed 's/"//g')

        while read IF; do
            ifIp=$(ip address show $IF | grep -w inet | awk -F ' ' '{print $2}' | awk -F '/' '{print $1}')
            # handle multiple IPs
            if [[ $(echo $ifIp | grep -c $privIp) -ne 0 ]]; then
                interfaceMap["eth$i"]=$IF
                break
            fi
        done < <($IP address show | egrep ^[0-9]: | awk -F ' ' '{print $2}' | sed 's/://' | grep -v lo)
    done

    ETH0=${interfaceMap['eth0']}
    ETH1=${interfaceMap['eth1']}
    ETH2=${interfaceMap['eth2']}
    if [[ $MODE == "single" ]]; then
        ETH3=${interfaceMap['eth3']}
        ETH4=${interfaceMap['eth4']}
    fi

    if [[ $MODE == "single" ]]; then
        # Gateway is second IP in CIDR
        ETH2_GW=$(getGatewayIp $ETH2)
        ETH3_GW=$(getGatewayIp $ETH3)
        ETH4_GW=$(getGatewayIp $ETH4)
        loggerHfe ""
        loggerHfe " Default GWs:"
        loggerHfe " ETH2_GW          $ETH2_GW"
        loggerHfe " ETH3_GW          $ETH3_GW"
        loggerHfe " ETH4_GW          $ETH4_GW"
    else
        # Gateway is second IP in CIDR
        ETH1_GW=$(getGatewayIp $ETH1)
        ETH2_GW=$(getGatewayIp $ETH2)
        loggerHfe ""
        loggerHfe " Default GWs:"
        loggerHfe " ETH1_GW          $ETH1_GW"
        loggerHfe " ETH2_GW          $ETH2_GW"
    fi

    #Set the MGMT interface information
    if [[ $MODE == "single" ]]; then
        MGMT=$ETH2
        MGMT_GW=$ETH2_GW
    else
        MGMT=$ETH1
        MGMT_GW=$ETH1_GW
    fi
}


readConfig()
{
    loggerHfe " Read variables from file $varFile"

    source $varFile
    SBC_PKT_PORT_NAME=$(echo $SBC_PKT_PORT_NAME | awk '{print toupper($0)}') #Force UpperCase
    loggerHfe " Data from $varFile:"
    if [[ $SBC_PKT_PORT_NAME == "" ]]; then
        MODE="single"
    else
        MODE="split"
        loggerHfe " SBC_PKT_PORT_NAME $SBC_PKT_PORT_NAME"
    fi
    loggerHfe " ACTIVE_SBC_VM_NAME $ACTIVE_SBC_VM_NAME"
    loggerHfe " STANDBY_SBC_VM_NAME $STANDBY_SBC_VM_NAME"
    loggerHfe " REMOTE_SSH_MACHINE_IP $REMOTE_SSH_MACHINE_IP"
    loggerHfe ""

    if [[ $MODE == "single" ]]; then
        loggerHfe " SBC_PKT_PORT_NAME is not configured. Assuming single HFE instance with 5 interfaces"
        loggerHfe ""
    fi
}


## Parameters passed in to configureNATRules
## $1 - Interface for Public/Private connection to HFE
## $2 - Interface to Pkt0/Pkt1 to Active SBC
## $3 - Gateway for Pkt0/Pkt1 to Active SBC
## $4 - Print configuration messages to HFE log
configureNATRules()
{
    ## Set forward ACCEPT rule for packets coming into HFE
    $IPTABLES -I FORWARD -i $1 -o $2 -j ACCEPT
    if [ $? -eq 0 ]; then
        [[ ! -z $4 ]] && loggerHfe " Set Forward ACCEPT rule all packets coming from outside $1 to $2 towards SBC"
    else
        errorAndExit "Failed to set forward ACCEPT rule for all packets coming on IP($1)"
    fi

    ## Set forward ACCEPT rule for packets coming from SBC
    $IPTABLES -I FORWARD -i $2 -o $1 -j ACCEPT
    if [ $? -eq 0 ]; then
        [[ ! -z $4 ]] && loggerHfe " Set Forward ACCEPT rule all packets coming from SBC ($2) to $1"
    else
        errorAndExit "Failed to set ACCEPT rule all packets coming from SBC ($2) to $1"
    fi

    if [[ $1 == $ETH0 ]]; then
        sbcIpCount="${#ACTIVE_SBC_IP_ARR[@]}"
        hfeIpCount="${#HFE_ETH0_IP_ARR[@]}"
        addNumRoute=$(( hfeIpCount < sbcIpCount ? hfeIpCount : sbcIpCount ))
    else
        sbcIpCount="${#ACTIVE_SBC_IP_PKT1_ARR[@]}"
        hfeIpCount="${#HFE_ETH1_IP_ARR[@]}"
        addNumRoute=$(( hfeIpCount < sbcIpCount ? hfeIpCount : sbcIpCount ))
    fi

    
    for (( idx=0; idx<$addNumRoute; idx++ ))
    do
        if [[ $1 == $ETH0 ]]; then
            hfeIp=${HFE_ETH0_IP_ARR[$idx]}
            sbcIp=${ACTIVE_SBC_IP_ARR[$idx]}
        else
            hfeIp=${HFE_ETH1_IP_ARR[$idx]}
            sbcIp=${ACTIVE_SBC_IP_PKT1_ARR[$idx]}
        fi

        ## Set DNAT for detination IP
        $IPTABLES -t nat -I PREROUTING -d $hfeIp -j DNAT --to $sbcIp
        if [ $? -eq 0 ]; then
            [[ ! -z $4 ]] && loggerHfe " Set up proper DNAT for destination IP $hfeIp to offset $sbcIp."
        else
            errorAndExit "Failed to set DNAT rule for destination IP $hfeIp to offset $sbcIp."
        fi

        ## Set SNAT for external interface to HFE
        $IPTABLES -t nat -I POSTROUTING -o $1 -s $sbcIp -j SNAT --to $hfeIp
        if [ $? -eq 0 ]; then
            [[ ! -z $4 ]] && loggerHfe " Set up POSTROUTING rule (source IP $sbcIp, to offset $hfeIp) for packet sent on $1"
        else
            errorAndExit "Failed to set POSTROUTING rule (source IP $sbcIp, to offset $hfeIp) for packet sent on $1"
        fi
    done
}

###################################################################################################
####                                         WARNING
####
####                Each call of this function will result in pkt drop.
####
#### Each conntrack flush operation has a associated time penalty (pkt drop), call this function
#### only when you are all set with setting up new iptables rules (and old rules are cleaned up).
####
####
####
###################################################################################################
clearOldRules()
{   
    ## Reset connection tracking
    ## Any packet received on input interfaces before NAT rules are set are not forwarded
    ## to sbc, connection tracking will not forward those packets even
    ## if NAT rules are set after receiving first packet from that source
    ## IP/Port as it has cache entry for source IP and Port.
    ## Reset connection tracking will treat them as new stream and those packets
    ## will be forwarded to SBC once SNAT and DNAT rules are setup
    ## properly.

    $CONNTRACK -F conntrack
    if [ $? -eq 0 ]; then
        [[ ! -z $6 ]] && loggerHfe " Flushing connection tracking rules."
    else
        [[ ! -z $6 ]] && loggerHfe " (WARNING):Flushing connection tracking rules failed."
    fi
}

configureMgmtNAT()
{
    loggerHfe " Optional configuration to reach $MGMT using Remote IP. "

    if [ -z "${REMOTE_SSH_MACHINE_IP// }" ]; then
        loggerHfe " No IP is given for REMOTE_SSH_MACHINE_IP field, no route is set for managing this instance over $MGMT"
    else
        loggerHfe " $MGMT is used to manage this HFE instance, we can login using private IP to manage HFE machine without setting default route"
        loggerHfe " default route points to eth0 which will be used to interface all traffic for SBC"

        IFS=',' read -ra SSH_IP_LIST <<< "$REMOTE_SSH_MACHINE_IP"
        for REMOTE_MACHINE_IP in "${SSH_IP_LIST[@]}"; do
            $IP route | grep -E "$REMOTE_MACHINE_IP.*$MGMT_GW.*$MGMT"
            if [ "$?" = "0" ]; then
                loggerHfe " Route is already available for remote machine's public IP($REMOTE_MACHINE_IP), from this IP you can SSH to HFE over Remote IP($MGMT)"
            else
                $IP route add $REMOTE_MACHINE_IP via $MGMT_GW dev $MGMT
                if [ "$?" = "0" ]; then
                    loggerHfe " Route added for remote machine's public IP($REMOTE_MACHINE_IP), from this IP you can SSH to HFE over Remote IP($MGMT)"
                else
                    errorAndExit "Failed to add route for ($REMOTE_MACHINE_IP)"
                fi
            fi
        done
    fi
}

showCurrentConfig()
{
    loggerHfe " Applied iptable rules and kernel routing table. "
    natTable=`$IPTABLES -t nat -vnL`

    loggerHfe " NAT tables:"
    loggerHfe " $natTable "

    filterTable=`$IPTABLES -t filter -vnL`

    loggerHfe " Filter tables:"
    loggerHfe " $filterTable "

    routeOutput=`$IP route`
    loggerHfe " Route:"
    loggerHfe " $routeOutput "
}

getSbcPktIps()
{
    local SBC_NAME=$1
    if [[ $2 == "PKT0" ]]; then
        IF="2"
    else
        IF="3"
    fi
    
    unsortedList=''
    sortedList=''

    vmInstance=$($CURL "https://management.azure.com/subscriptions/$SUB_ID/resourceGroups/$RG_NAME/providers/Microsoft.Compute/virtualMachines/$SBC_NAME?$apiversion" -H "$authhead")
    networkInterfaceId=$(echo $vmInstance | $JQ .properties.networkProfile.networkInterfaces[$IF].id | sed 's/"//g')

    # Get Primary IPs or Secondary IPs
    ipConfigs=$($CURL "https://management.azure.com$networkInterfaceId?$apiversion" -H "$authhead" | $JQ .properties.ipConfigurations)
    configCount=$(echo $ipConfigs | $JQ . | $JQ length)
    for i in $(seq 0 $((configCount-1))); do
        if [[ $(echo $ipConfigs | $JQ .[$i].properties.primary | sed 's/"//g') == "true" ]]; then
            #Don't use primary on SBC
            continue
        else
            ip=$(echo $ipConfigs | $JQ .[$i].properties.privateIPAddress | sed 's/"//g')
            unsortedList+="$ip "
        fi
    done

    sortedList=$(echo $unsortedList| tr " " '\n' | sort -n -t . -k1,1 -k2,2 -k 3,3 -k4,4)

    if [[ -z "$sortedList" ]];then
        errorAndExit "Failed to get IP(s) assigned to $2 on $1."
    fi
    loggerHfe " $SBC_NAME - IPs for $2 are $(echo $sortedList | tr '\n' " ")"
    echo $sortedList
}

getHfeIps()
{
    interface=$1
    unsortedList=''
    sortedList=''

    if [[ $interface == $ETH0 ]]; then
        nic=0
    else
        nic=1
    fi

    vmInstance=$($CURL "https://management.azure.com/subscriptions/$SUB_ID/resourceGroups/$RG_NAME/providers/Microsoft.Compute/virtualMachines/$HFE_NAME?$apiversion" -H "$authhead")
    networkInterfaceId=$(echo $vmInstance | $JQ .properties.networkProfile.networkInterfaces[$nic].id | sed 's/"//g')

    ipConfigs=$($CURL "https://management.azure.com$networkInterfaceId?$apiversion" -H "$authhead" | $JQ .properties.ipConfigurations)
    configCount=$(echo $ipConfigs | $JQ . | $JQ length)
    for i in $(seq 0 $((configCount-1))); do
        ip=$(echo $ipConfigs | $JQ .[$i].properties.privateIPAddress | sed 's/"//g')
        unsortedList+="$ip "
    done

    sortedList=$(echo $unsortedList| tr " " '\n' | sort -n -t . -k1,1 -k2,2 -k 3,3 -k4,4)

    if [[ -z "$sortedList" ]];then
        errorAndExit "Failed to get IP(s) assigned to $HFE_NAME on $interface."
    fi
    loggerHfe " $HFE_NAME - IPs for $interface are $(echo $sortedList | tr '\n' " ")"
    echo $sortedList
}

setLogRotate()
{
    # Setup HFE logRotate to backup and rotate logs which report status of connection between HFE and Active SBC

    if [ ! -f $hfeLogRotateConf ]; then
        echo -e "$statusLogFile" >> $hfeLogRotateConf
        echo -e "{" >> $hfeLogRotateConf
        echo -e "    rotate 4" >> $hfeLogRotateConf
        echo -e "    size=250M" >> $hfeLogRotateConf
        echo -e "    missingok" >> $hfeLogRotateConf
        echo -e "    notifempty" >> $hfeLogRotateConf
        echo -e "    dateformat -%d%m%Y" >> $hfeLogRotateConf
        echo -e "    compress" >> $hfeLogRotateConf
        echo -e "}" >> $hfeLogRotateConf
    fi
}

checkSbcConnectivity()
{
    local switched=0
    local counter=0
    local verifyCount=0
    while :; do
############################################### CAUTION #################################
#
#        -t (value) < -P (value)
#
#########################################################################################
#      -c, --count=N
#
#        Number of request packets to send to each target. In this mode, a line is
#        displayed for each received response (this can suppressed with -q or -Q). Also,
#        statistics about responses for each target are displayed when all requests have
#        been sent (or when interrupted).
#
#
#      -p, --period= MSEC
#
#        In looping or counting modes (-l, -c, or -C), this parameter sets the time in
#        milliseconds that fping waits between successive packets to an individual
#        target. Default is 1000 and minimum is 10.
#
#
#
#      -t, --timeout= MSEC
#
#        Initial target timeout in milliseconds. In the default, non-loop mode,
#        the default timeout is 500ms, and it represents the amount of time that
#        fping waits for a response to its first request. Successive timeouts are
#        multiplied by the backoff factor specified with -B.
#
#        In loop/count mode, the default timeout is automatically adjusted to
#        match the "period" value (but not more than 2000ms). You can still
#        adjust the timeout value with this option, if you wish to, but note that
#        setting a value larger than "period" produces inconsistent results,
#        because the timeout value can be respected only for the last ping.
#
#        Also note that any received replies that are larger than the timeout
#        value, will be discarded.
############################################################################################

        # Use first IP
        $FPING -c 3 -t 100 -p 200 ${ACTIVE_SBC_IP_ARR[0]} &> /dev/null
        if [ $? -ne 0 ]
        then
            if [ $switched -eq 0 ]; then
                statusLoggerHfe "Connection error detected to Active SBC: $ACTIVE_SBC_VM_NAME. ${ACTIVE_SBC_IP_ARR[0]} did not respond. Attempting switchover."
                switched=1
            elif [ $switched -eq 1 ] && [ $(($counter % 10)) -eq 0 ]; then
                statusLoggerHfe "Connection error ongoing - No connection to SBC $SBC_PKT_PORT_NAME from HFE"
            fi
            
            # If single HFE (2.0), this is PKT0 config
            TEMP_SBC_IP_ARR=( ${ACTIVE_SBC_IP_ARR[@]} )
            ACTIVE_SBC_IP_ARR=( ${STANDBY_SBC_IP_ARR[@]} )
            STANDBY_SBC_IP_ARR=( ${TEMP_SBC_IP_ARR[@]} )

            TEMP_SBC_NAME=$ACTIVE_SBC_VM_NAME
            ACTIVE_SBC_VM_NAME=$STANDBY_SBC_VM_NAME
            STANDBY_SBC_VM_NAME=$TEMP_SBC_NAME

            if [[ $MODE == "single" ]]; then
                TEMP_SBC_IP_PKT1_ARR=( ${ACTIVE_SBC_IP_PKT1_ARR[@]} )
                ACTIVE_SBC_IP_PKT1_ARR=( ${STANDBY_SBC_IP_PKT1_ARR[@]} )
                STANDBY_SBC_IP_PKT1_ARR=( ${TEMP_SBC_IP_PKT1_ARR[@]} )
            fi

            statusLoggerHfeDebug "Before clearOldIptableRules"
            clearOldIptableRules
            statusLoggerHfeDebug "After clearOldIptableRules, and just before configureNATRules"

            statusLoggerHfeDebug "Verifying Routes"
            verifyRoutes
            verifyCount=0

            if [[ $MODE == "single" ]]; then
                configureNATRules $ETH0 $ETH3 $ETH3_GW
                configureNATRules $ETH1 $ETH4 $ETH4_GW
            else
                configureNATRules $ETH0 $ETH2 $ETH2_GW
            fi

            statusLoggerHfeDebug "After configureNATRules and just before clearOldRules"
            clearOldRules
            statusLoggerHfeDebug "After clearOldRules"
            
            let counter=$counter+1 #increment log count
            statusLoggerHfeDebug "Wait for SBC to complete switchover."
            statusLoggerHfeDebug ""
            statusLoggerHfeDebug ""
            sleep 2s
	    else
            if [ $INIT_LOG -eq 1 ]; then
                statusLoggerHfe "Initial HFE startup configuration complete. Successfully connected to $ACTIVE_SBC_VM_NAME"
                INIT_LOG=0
                switched=0
                counter=0
            elif [ $switched -eq 1 ]; then
                statusLoggerHfe "Switchover from old Active $TEMP_SBC_NAME to new Active $ACTIVE_SBC_VM_NAME complete. Connection established."
                switched=0
				counter=0
            fi
            statusLoggerHfeDebug "Nothing needs to be done, active is up $ACTIVE_SBC_VM_NAME"
	        sleep 0.5s
        fi
        
        # Verify Routes every ~10 secs
        if [ $verifyCount -eq 20 ]; then
            verifyRoutes
            verifyCount=0
        fi
        let verifyCount=$verifyCount+1

    done
}

# Verify authorization is successful
verifyAutorization()
{
    local SBC_NAME=$1 
    local count=0

    loggerHfe " Validating authorization token"

    vmResult=$($CURL "https://management.azure.com/subscriptions/$SUB_ID/resourceGroups/$RG_NAME/providers/Microsoft.Compute/virtualMachines/$SBC_NAME?$apiversion" -H "$authhead" | $JQ .error.code)
    while [[ $vmResult != "null" ]] && [ $count -lt 30 ]; do
        loggerHfe " Authorization validation for attempt $count failed. Retrying in 30 seconds"
        let count=$count+1
        sleep 30s
        token=$($CURL "http://169.254.169.254/metadata/identity/oauth2/token?$apiversion&$resource" -H Metadata:true | $JQ .access_token | sed 's/"//g')
        authhead="Authorization:Bearer $token"
        vmResult=$($CURL "https://management.azure.com/subscriptions/$SUB_ID/resourceGroups/$RG_NAME/providers/Microsoft.Compute/virtualMachines/$SBC_NAME?$apiversion" -H "$authhead" | $JQ .error.code)
    done
    
    if [ $count -eq 30 ]; then
        errorAndExit "Authorization still failing after 15 minutes. Verify identity roles and reboot! Exiting"
    else
        loggerHfe " Authorization validation successful for attempt $count. Continuing to get SBC information"
    fi
}

getAzureVars()
{
    metaDataJson=$($CURL "http://169.254.169.254/metadata/instance/compute?$apiversion" -H Metadata:true)
    RG_NAME=$(echo $metaDataJson | $JQ .resourceGroupName | sed 's/"//g')
    SUB_ID=$(echo $metaDataJson | $JQ .subscriptionId | sed 's/"//g')
    HFE_NAME=$(echo $metaDataJson | $JQ .name | sed 's/"//g')

    token=$($CURL "http://169.254.169.254/metadata/identity/oauth2/token?$apiversion&$resource" -H Metadata:true | $JQ .access_token | sed 's/"//g')
    authhead="Authorization:Bearer $token"
}

extractHfeConfigIps()
{
    # Get IPs
    if [[ $MODE == "single" ]]; then
        ACTIVE_SBC_IP_ARR=( $(getSbcPktIps $ACTIVE_SBC_VM_NAME "PKT0") )
        STANDBY_SBC_IP_ARR=( $(getSbcPktIps $STANDBY_SBC_VM_NAME "PKT0") )
        HFE_ETH0_IP_ARR=( $(getHfeIps $ETH0) )
        
        ACTIVE_SBC_IP_PKT1_ARR=( $(getSbcPktIps $ACTIVE_SBC_VM_NAME "PKT1") )
        STANDBY_SBC_IP_PKT1_ARR=( $(getSbcPktIps $STANDBY_SBC_VM_NAME "PKT1") )
        HFE_ETH1_IP_ARR=( $(getHfeIps $ETH1) )
    else
        ACTIVE_SBC_IP_ARR=( $(getSbcPktIps $ACTIVE_SBC_VM_NAME $SBC_PKT_PORT_NAME) )
        STANDBY_SBC_IP_ARR=( $(getSbcPktIps $STANDBY_SBC_VM_NAME $SBC_PKT_PORT_NAME) )
        HFE_ETH0_IP_ARR=( $(getHfeIps $ETH0) )
    fi
    
    # Verify we have everything
    if [ -z $ACTIVE_SBC_IP_ARR ] || [ -z $STANDBY_SBC_IP_ARR ] || [ -z $HFE_ETH0_IP_ARR ] ; then
        errorAndExit "SBC information missing. Exiting"
    elif [[ $MODE == "single" ]]; then
        if [ -z $ACTIVE_SBC_IP_PKT1_ARR ] || [ -z $STANDBY_SBC_IP_PKT1_ARR ] || [ -z $HFE_ETH1_IP_ARR ]; then
            errorAndExit "SBC information missing. Exiting"
        fi
    fi
}

# Returns interface type.
# SLAVE interface will have symlink to it's master.
# if the symlink is present, then it's a salve interface.
# Normal interface will not have any of these files.
getIntfType()
{
    local intf=$1

    if ls /sys/class/net/$intf/lower_* 1> /dev/null 2>&1; then
        gRetVal=$INTF_TYPE_MASTER
        return 0
    fi

    if ls /sys/class/net/$intf/master  1> /dev/null 2>&1; then
        gRetVal=$INTF_TYPE_SLAVE
        return 0
    fi

    gRetVal=$INTF_TYPE_NORMAL
    return 0
}

function setSecondaryIntfQueueLen()
{
    local netdev=""
    local intfType=$INTF_TYPE_NORMAL
    local rxPresetMax=4096
    local txPresetMax=4096
    local rxQLen=0
    local txQLen=0


    for netdev in $(ls /sys/class/net); do
        if [ "$netdev" != "lo" ]; then
            getIntfType $netdev
            intfType=$gRetVal
            if [ $intfType -eq $INTF_TYPE_SLAVE ]; then
                rxQLen=$(ethtool -g $netdev| grep -A 4 "Current hardware settings" | grep RX: | cut -d ':' -f2| xargs)
                if [ $? -ne 0 ]; then
                    errorAndExit "$netdev: Failed to get Current RX queue length"
                fi
                txQLen=$(ethtool -g $netdev| grep -A 4 "Current hardware settings" | grep TX: | cut -d ':' -f2| xargs)
                if [ $? -ne 0 ]; then
                    errorAndExit "$netdev: Failed to get Current TX queue length"
                fi

                #find the preset Max RX queue length
                rxPresetMax=$(ethtool -g $netdev| grep -A 4 "Pre-set maximums" | grep RX: | cut -d ':' -f2| xargs)
                if [ $? -ne 0 ]; then
                    errorAndExit "$netdev: Failed to get Pre-set Maximum RX queue length"
                fi
                #restrict the queue length to Max 8192
                rxPresetMax=$((rxPresetMax>MAX_QUEUE_LENGTH ? MAX_QUEUE_LENGTH : rxPresetMax))

                #find the preset Max TX queue length
                txPresetMax=$(ethtool -g $netdev | grep -A 4 "Pre-set maximums" | grep TX: | cut -d ':' -f2| xargs)
                if [ $? -ne 0 ]; then
                    errorAndExit "$netdev: Failed to get Pre-set Maximum TX queue length"
                fi
                #restrict the queue length to Max 8192
                txPresetMax=$((txPresetMax>MAX_QUEUE_LENGTH ? MAX_QUEUE_LENGTH : txPresetMax))

                if [ $rxQLen -lt $rxPresetMax ]; then
                    loggerHfe " $netdev: changing RX queue length from $rxQLen to $rxPresetMax"
                    ethtool -G $netdev rx $rxPresetMax
                fi
                if [ $txQLen -lt $txPresetMax ]; then
                    loggerHfe " $netdev: changing TX queue length from $txQLen to $txPresetMax"
                    ethtool -G $netdev tx $txPresetMax
                fi
            fi
        fi
    done

    sleep 1
}


main()
{
    case $1 in
        "setup") 
            prepareHFEInstance
            preConfigure
            readConfig
            getAzureVars
            verifyAutorization $ACTIVE_SBC_VM_NAME
            setInterfaceMap
            extractHfeConfigIps
            if [[ $MODE == "single" ]]; then
                # Configure Routes for Active SBC PKT0
                configureSubnetRoute $ACTIVE_SBC_VM_NAME "PKT0" $ETH3_GW $ETH3
                configureNATRules $ETH0 $ETH3 $ETH3_GW 1
                # Configure Routes for Active SBC PKT1
                configureSubnetRoute $ACTIVE_SBC_VM_NAME "PKT1" $ETH4_GW $ETH4
                configureNATRules $ETH1 $ETH4 $ETH4_GW 1
            else
                # Configure Routes for Active SBC
                configureSubnetRoute $ACTIVE_SBC_VM_NAME $SBC_PKT_PORT_NAME $ETH2_GW $ETH2
                configureNATRules $ETH0 $ETH2 $ETH2_GW 1
            fi
            clearOldRules
            configureMgmtNAT
            setSecondaryIntfQueueLen
            showCurrentConfig
            checkSbcConnectivity
            ;;
        "cleanup")
            prepareHFEInstance
            preConfigure
            readConfig
            getAzureVars
            setInterfaceMap
            extractHfeConfigIps
            routeCleanUp
            doneMessage
            ;;
        *) 
            usage "Unrecognized switch"
            ;;
    esac
}

[[ $# -ne 1 ]] && usage

main $1

Create a file Name: azureSbcHFEFullCreate_21.sh, paste the code below and Update the values from your current Azure tenant till line# 78.

• Sub: Subscription ID of your Azure Tenant
• Boot Diagnostic Store Name:

#!/bin/bash


#########
#
# Script to create a Full HFE setup in Azure
#
#########


# Remove this if already logged in.
#az login


sub="<your subscription id e.g >"
bootDiags="<your boot diag storage e.g. connxtestsbcdiagstore>"
rg="<Resource Group Name, eg SWE-HFE-Test>"
instanceBaseName="<instance name e.g. testsbc-hfe>"
instanceName1="$instanceBaseName-act"
instanceName2="$instanceBaseName-sby"
imgName="<base Image , reach out to your Ribbon SE for the URL to copy it your subscription and after copy put your tenant URL e.g./subscriptions/aad0f841-bfd5-430b-8781-146e1bea0e84/resourceGroups/SWE-HFE-Test/providers/Microsoft.Compute/images/rbbn-sbc-v09.02.00.img>"
sshKeyFile="azureSshKey.pub"
userAssignedIdentity="/subscriptions/aad0f841-bfd5-430b-8781-146e1bea0e84/resourcegroups/SWE-HFE-Test/providers/Microsoft.ManagedIdentity/userAssignedIdentities/rbbnUami"
location="eastus" #e.g. Central US
vmSize="Standard_DS3_v2" # Standard_D8s_v3 has 4cpus/14GB RAM/4 NICS
diskSizeGB="65" # 65


### UPDATE TO 1 TO CREATE SECONDARY IP CONFIGS ###
secondaryFlag=1


## Network Settings (MUST BE PRE-CREATED) ##
vnet="TEST-USE1-vnet"
mgtsub="mgmt"
hasub="HA"
pkt0sub="sbc-pkt0"
pkt1sub="sbc-pkt1"
sgn="RbbnSbcSG"


## SBC USERDATA SETTINGS ##
userDataAct="userDataAct.json"
userDataSby="userDataSby.json"
ActiveCeName="connxtestsbc1a"
StandbyCeName="connxtestsbc1b"
SystemName="connxtestsbc1"
SbcPersonalityType="isbc" #isbc
AdminSshKey="$(cat $sshKeyFile)"
ReverseNatPkt0="True"
ReverseNatPkt1="True"


## SBC Static IPs ##
actHaIp="10.51.10.4"
sbyHaIp="10.51.10.5"


## HFE VARS ##
hfeUserData="hfeUserData_21.txt"
hfeName="$instanceBaseName-hfn"
sshIp="73.226.201.242,63.241.104.84"
azFile="https://connxrbbnhfestorage.blob.core.windows.net/connx-hfescripts-container/HFE_AZ.sh"
pkt0Src="13.82.49.74/32" #CREATES THE ROUTE FOR SENDING TRAFFIC BACK FROM SBC
pkt1Src="10.100.250.0/29" #CREATES THE ROUTE FOR SENDING TRAFFIC BACK FROM SBC
hfepubsub="hfe-pub"
hfeprivsub="hfe-pvt"
hfepkt0sub="hfe-pkt0"
hfepkt1sub="hfe-pkt1"
hfeUser="rbbn" #rbbn
hfeSize="Standard_DS3_v2" #e.g Standard_A8m_v2
hfeImg="Canonical:UbuntuServer:18.04-LTS:latest"



## HFE STATIC IPS ##
hfePkt0OutIp="10.51.60.4"
hfePkt1OutIp="10.51.70.4"



if [ ! -f $hfeUserData ]; then
echo"$hfeUserData does not exist. Exiting..."
exit 1
fi


# Write out SBC UserData
echo -e "{
\"CERole\" : \"ACTIVE\",
\"CEName\": \"$ActiveCeName\",
\"ReverseNatPkt0\" : \"$ReverseNatPkt0\",
\"ReverseNatPkt1\" : \"$ReverseNatPkt1\",
\"SystemName\": \"$SystemName\",
\"PeerCEHa0IPv4Address\" : \"$sbyHaIp\",
\"ClusterIp\" : \"$sbyHaIp\",
\"PeerCEName\" : \"$StandbyCeName\",
\"SbcPersonalityType\": \"$SbcPersonalityType\",
\"SbcHaMode\" : \"1to1\",
\"AdminSshKey\" : \"$AdminSshKey\",
\"ThirdPartyCpuAlloc\" : \"0\",
\"ThirdPartyMemAlloc\" : \"0\",
\"PeerInstanceName\" : \"$instanceName2\",
\"Pkt0HfeInstanceName\" : \"$hfeName-pkt0\",
\"Pkt1HfeInstanceName\" : \"$hfeName-pkt1\"
}" > $userDataAct


echo -e "{
\"CERole\" : \"STANDBY\",
\"CEName\": \"$StandbyCeName\",
\"ReverseNatPkt0\" : \"$ReverseNatPkt0\",
\"ReverseNatPkt1\" : \"$ReverseNatPkt1\",
\"SystemName\": \"$SystemName\",
\"PeerCEHa0IPv4Address\" : \"$actHaIp\",
\"ClusterIp\" : \"$actHaIp\",
\"PeerCEName\" : \"$ActiveCeName\",
\"SbcPersonalityType\": \"$SbcPersonalityType\",
\"SbcHaMode\" : \"1to1\",
\"AdminSshKey\" : \"$AdminSshKey\",
\"ThirdPartyCpuAlloc\" : \"0\",
\"ThirdPartyMemAlloc\" : \"0\",
\"PeerInstanceName\" : \"$instanceName1\",
\"Pkt0HfeInstanceName\" : \"$hfeName-pkt0\",
\"Pkt1HfeInstanceName\" : \"$hfeName-pkt1\"
}" > $userDataSby


#HFE USER DATA UPDATE
cp -f $hfeUserData $hfeUserData.update
sed -i "s~<HFE_SCRIPT_LOCATION>~$azFile~" $hfeUserData.update
sed -i "s/<ACTIVE_SBC_NAME>/$instanceName1/" $hfeUserData.update
sed -i "s/<STANDBY_SBC_NAME>/$instanceName2/" $hfeUserData.update
sed -i "s/<REMOTE_SSH_MACHINE_IP>/$sshIp/" $hfeUserData.update


cp $hfeUserData.update $hfeUserData.update.pkt0
sed -i "s/<SBC_PKT_PORT_NAME>/PKT0/" $hfeUserData.update.pkt0
cp $hfeUserData.update $hfeUserData.update.pkt1
sed -i "s/<SBC_PKT_PORT_NAME>/PKT1/" $hfeUserData.update.pkt1



#Create public IP for mgmt0
az network public-ip create -g $rg -n "$instanceName1-mgt-ip" --allocation-method Static --idle-timeout 10 --subscription $sub
az network public-ip create -g $rg -n "$instanceName2-mgt-ip" --allocation-method Static --idle-timeout 10 --subscription $sub



#Create NICs
az network nic create --name "$instanceName1-mgt" --resource-group $rg --vnet-name $vnet --subnet $mgtsub --network-security-group $sgn --public-ip-address "$instanceName1-mgt-ip" --subscription $sub
az network nic create --name "$instanceName1-ha" --resource-group $rg --vnet-name $vnet --subnet $hasub --network-security-group $sgn --private-ip-address $actHaIp --subscription $sub
az network nic create --name "$instanceName1-pkt0" --resource-group $rg --vnet-name $vnet --subnet $pkt0sub --network-security-group $sgn --ip-forwarding --subscription $sub
az network nic create --name "$instanceName1-pkt1" --resource-group $rg --vnet-name $vnet --subnet $pkt1sub --network-security-group $sgn --ip-forwarding --subscription $sub
az network nic create --name "$instanceName2-mgt" --resource-group $rg --vnet-name $vnet --subnet $mgtsub --network-security-group $sgn --public-ip-address "$instanceName2-mgt-ip" --subscription $sub
az network nic create --name "$instanceName2-ha" --resource-group $rg --vnet-name $vnet --subnet $hasub --network-security-group $sgn --private-ip-address $sbyHaIp --subscription $sub
az network nic create --name "$instanceName2-pkt0" --resource-group $rg --vnet-name $vnet --subnet $pkt0sub --network-security-group $sgn --ip-forwarding --subscription $sub
az network nic create --name "$instanceName2-pkt1" --resource-group $rg --vnet-name $vnet --subnet $pkt1sub --network-security-group $sgn --ip-forwarding --subscription $sub



#Create secondary IPs
az network nic ip-config create --name "$instanceName1-pkt0-alias" --nic-name "$instanceName1-pkt0" --resource-group $rg --subscription $sub
az network nic ip-config create --name "$instanceName1-pkt1-alias" --nic-name "$instanceName1-pkt1" --resource-group $rg --subscription $sub
az network nic ip-config create --name "$instanceName2-pkt0-alias" --nic-name "$instanceName2-pkt0" --resource-group $rg --subscription $sub
az network nic ip-config create --name "$instanceName2-pkt1-alias" --nic-name "$instanceName2-pkt1" --resource-group $rg --subscription $sub


#HFE ROUTETABLES
az network route-table create -g $rg -n "$hfeName-pkt0-route-table" --subscription $sub
az network route-table create -g $rg -n "$hfeName-pkt1-route-table" --subscription $sub
az network route-table route create -g $rg -n "$hfeName-pkt0-route" --address-prefix $pkt0Src --next-hop-type VirtualAppliance --route-table-name "$hfeName-pkt0-route-table" --next-hop-ip-address $hfePkt0OutIp --subscription $sub
az network route-table route create -g $rg -n "$hfeName-pkt1-route" --address-prefix $pkt1Src --next-hop-type VirtualAppliance --route-table-name "$hfeName-pkt1-route-table" --next-hop-ip-address $hfePkt1OutIp --subscription $sub


az network vnet subnet update --resource-group $rg --vnet-name $vnet --name $pkt0sub --route-table "$hfeName-pkt0-route-table" --subscription $sub
az network vnet subnet update --resource-group $rg --vnet-name $vnet --name $pkt1sub --route-table "$hfeName-pkt1-route-table" --subscription $sub



#HFE public
az network public-ip create -g $rg -n "$hfeName-pub-ip" --allocation-method Static --idle-timeout 10 --subscription $sub
az network public-ip create -g $rg -n "$hfeName-pkt0-mgt-ip" --allocation-method Static --idle-timeout 10 --subscription $sub
az network public-ip create -g $rg -n "$hfeName-pkt1-mgt-ip" --allocation-method Static --idle-timeout 10 --subscription $sub


#HFE NICS
az network nic create --name "$hfeName-pkt0In" --resource-group $rg --vnet-name $vnet --subnet $hfepubsub --network-security-group $sgn --public-ip-address "$hfeName-pub-ip" --ip-forwarding --subscription $sub
az network nic create --name "$hfeName-pkt1In" --resource-group $rg --vnet-name $vnet --subnet $hfeprivsub --network-security-group $sgn --ip-forwarding --subscription $sub
az network nic create --name "$hfeName-pkt0-mgt" --resource-group $rg --vnet-name $vnet --subnet $mgtsub --network-security-group $sgn --public-ip-address "$hfeName-pkt0-mgt-ip" --subscription $sub
az network nic create --name "$hfeName-pkt1-mgt" --resource-group $rg --vnet-name $vnet --subnet $mgtsub --network-security-group $sgn --public-ip-address "$hfeName-pkt1-mgt-ip" --subscription $sub
az network nic create --name "$hfeName-pkt0Out" --resource-group $rg --vnet-name $vnet --subnet $hfepkt0sub --network-security-group $sgn --private-ip-address $hfePkt0OutIp --ip-forwarding --subscription $sub
az network nic create --name "$hfeName-pkt1Out" --resource-group $rg --vnet-name $vnet --subnet $hfepkt1sub --network-security-group $sgn --private-ip-address $hfePkt1OutIp --ip-forwarding --subscription $sub



#CREATE pkt0 HFE NODE
az vm create --name $hfeName-pkt0 --resource-group $rg --admin-username $hfeUser --custom-data $hfeUserData.update.pkt0 --image $hfeImg \
--location "$location" --size $hfeSize \
--ssh-key-values $sshKeyFile --nics "$hfeName-pkt0In" "$hfeName-pkt0-mgt" "$hfeName-pkt0Out" \
--boot-diagnostics-storage $bootDiags --assign-identity $userAssignedIdentity --subscription $sub &


#CREATE pkt1 HFE NODE
az vm create --name $hfeName-pkt1 --resource-group $rg --admin-username $hfeUser --custom-data $hfeUserData.update.pkt1 --image $hfeImg \
--location "$location" --size $hfeSize \
--ssh-key-values $sshKeyFile --nics "$hfeName-pkt1In" "$hfeName-pkt1-mgt" "$hfeName-pkt1Out" \
--boot-diagnostics-storage $bootDiags --assign-identity $userAssignedIdentity --subscription $sub &



echo "HFE Created"
sleep 20s


#Create SBCs VM
az vm create --name $instanceName1 --resource-group $rg --admin-username linuxadmin --custom-data $userDataAct --image $imgName \
--location "$location" --os-disk-size-gb $diskSizeGB --size $vmSize --ssh-dest-key-path /home/linuxadmin/.ssh/authorized_keys \
--ssh-key-values $sshKeyFile --nics "$instanceName1-mgt" "$instanceName1-ha" "$instanceName1-pkt0" "$instanceName1-pkt1" --boot-diagnostics-storage $bootDiags \
--assign-identity $userAssignedIdentity --subscription $sub&



sleep 15s


az vm create --name $instanceName2 --resource-group $rg --admin-username linuxadmin --custom-data $userDataSby --image $imgName \
--location "$location" --os-disk-size-gb $diskSizeGB --size $vmSize --ssh-dest-key-path /home/linuxadmin/.ssh/authorized_keys \
--ssh-key-values $sshKeyFile --nics "$instanceName2-mgt" "$instanceName2-ha" "$instanceName2-pkt0" "$instanceName2-pkt1" --boot-diagnostics-storage $bootDiags \
--assign-identity $userAssignedIdentity --subscription $sub &


echo "Done"