#!/bin/sh

PROD=$0
cnt=2
size=160
idle_limit=10 #percent
avg_limit=100 #ms
tmpfile=/tmp/rmon.$$
logfile=/var/log/nsg_error_events.log


trap on_die TERM INT EXIT

on_die()
{
	rm -f $tmpfile
	exit 0
}

logit_debug()
{
        local data="$1"
	local date=`date +"%b %d %H:%M:%S"` 

	if [ ! -e /tmp/nsg_route_debug ]; then
		return;
	fi

        echo "$PROD: $data"

        logger "$PROD: $data"
}

logit()
{
        local data="$1"
	local date=`date +"%b %d %H:%M:%S"` 

        echo "$PROD: $data"

        logger "$PROD: $data"
}

logit_crit()
{
        local data="$1"
	local date=`date +"%b %d %H:%M:%S"` 

        echo "$PROD: $data"

	echo "$date: $data" > $logfile.$$
	cat $logfile >> $logfile.$$ 
	mv $logfile.$$ $logfile

        logger "$PROD: $data"
}


runit()
{
        local run="$1"
        logit "$run";

        eval "$run";
        if [ $? -ne 0 ]; then
                logit "Error: Failed $?";
                return 1

        fi

        return 0;
}



run_host_check()
{
	local host=$1
	local host2=$2
	local err=0;
	logit_debug "Testing $host"
	avg_f=`ping -c $cnt -w 2 -W 1500 -s $size -q -Q 0x18 $host | grep rtt | cut -d'/' -f 5`
	if [ "$avg_f" = "" ]; then
		err=$((err+1))
		err_str="Warning: Failed ping on $host"
	fi
	logit_debug "Testing $host2"
	avg_f=`ping -c $cnt -w 2 -W 1500 -s $size -q -Q 0x18 $host2 | grep rtt | cut -d'/' -f 5`
	if [ "$avg_f" = "" ]; then
		err=$((err+1))
		err_str="Warning: Failed ping on $host2"
	fi

	logit_debug "Done Testing $host2"
	if [ $err -eq 1 ]; then
		logit "$err_str"
	fi	

	if [ $err -gt 1 ]; then
		return 1
	fi

	return 0;
}

ntp_restart()
{

        /etc/init.d/ntpd stop
        sleep 1
        eval "ntpdate 0.centos.pool.ntp.org"
        if [ $? -ne 0 ]; then
                eval "ntpdate 1.centos.pool.ntp.org"
                if [ $? -ne 0 ]; then
                        logit "Error: running nptdate";
                fi
        fi
        /etc/init.d/ntpd start
}


check_eth_ip()
{
	local dev=$1

	#In this case the route table for dev does not have default route 
  	#We must bounce this device and hope that default route and routing
	#table are recovered by ifup or ip-up scripts
	rc=`ifconfig $dev | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'`
	if [ "$rc" != "" ]; then
		logit "Warning: $dev has ip but routing table no default route: bouncing $dev"
		ifdown $dev; ifup $dev
	fi	

	return 0;	
}

resolv_check()
{
	rc=`diff -q /etc/resolv.conf /etc/resolv.conf.orig`

	if [ "$rc" != "" ]; then
		logit_crit "Warning: recovering resolv.conf from orig"
		remount rw web
		cp  /etc/resolv.conf.orig  /etc/resolv.conf
		remount ro web
	fi

}


check_for_multiple_default_routes()
{

	def_route_eth0=`ip route show table eth0 | grep default | cut -d' ' -f3`
	def_route_eth1=`ip route show table eth1 | grep default | cut -d' ' -f3`


	route_eth1=down
	route_eth0=down

	eth1_dev=eth1
	eth0_dev=eth0


	if [ "$def_route_eth1" = "" ]; then
		logit_debug "Route not available for 8.8.4.4"
		route_eth1=down
		check_eth_ip eth1
	else

		eth1_gw=$(ip route show table eth1 | grep -v tun |  awk '$3 ~ /^[1-9]+/ {print $3;}')
                eth1_dev=$(ip route show table eth1 | grep -v tun |  awk '$2 ~ /via/ && $4 ~ /dev/ {print $5;}')
		
		logit_debug "eth1_gw $eth1_gw  eth1_dev=$eth1_dev"

		run_host_check 8.8.4.4 4.2.2.5
		if [ $? -eq 0 ]; then
			route_eth1=up
			logit_debug "Ping check ok for 8.8.4.4"
		else
			logit_debug "Ping check failed for 8.8.4.4"
			route_eth1=down
		fi
	fi
	
	if [ "$def_route_eth0" = "" ]; then
		logit_debug "Route not available for 8.8.8.8"
		route_eth0=down
		check_eth_ip eth0
	else
		eth0_gw=$(ip route show table eth0 | grep -v tun |  awk '$3 ~ /^[1-9]+/ {print $3;}')
        	eth0_dev=$(ip route show table eth0 | grep -v tun |  awk '$2 ~ /via/ && $4 ~ /dev/ {print $5;}')

		logit_debug "eth0_gw $eth0_gw  eth0_dev=$eth0_dev"

		run_host_check 8.8.8.8 4.2.2.6
		if [ $? -eq 0 ]; then
			route_eth0=up
			logit_debug "Ping check ok for 8.8.8.8"
		else
			logit_debug "Ping check failed for 8.8.8.8"
			route_eth0=down
		fi
	fi


	if [ $route_eth0 = "down" ] && [ $route_eth1 = "down" ]; then
		#Nothing to do both routes are down
		logit_debug "Error: eth0 & eth1 routes down. Nothing to do."
		return 1
	fi

	restart_if=0;
	
	if [ $route_eth0 = "up" ] && [ $route_eth1 = "up" ]; then
		logit_debug "Ok is set to 2 recover both routes"
		rc=`ip route show | grep "nexthop via"`
		rc2=`ip route show | grep "default"`
		if [ "$rc" = "" ] || [ "$rc2" = "" ]; then
			runit "ip route del default"
			runit "ip route add default scope global nexthop via $eth0_gw dev $eth0_dev weight 1 nexthop via $eth1_gw dev $eth1_dev weight 1 "
			runit "ip route flush cache"
			logit_crit "eth0 and eth1 dual route recovered"
			ntp_restart
		fi

	fi

	if [ $route_eth1 = "down" ]; then
		logit_debug "route 8.8.4.4 not ok"
		rc=`ip route show | grep "nexthop via"`
		rc1=`ip route show | grep "default.*$eth1_dev"`
		rc2=`ip route show | grep "default"`
		if [ "$rc" != "" ] || [ "$rc1" != "" ] || [ "$rc2" = "" ]; then
			logit_crit "Error: eth1 route failed: $def_route_eth1. Taking down the route"
			runit "ip route del default"
			runit "route add default gw $def_route_eth0"
			runit "ip route flush cache"
			ntp_restart
		fi

		#At this point no use checing eth0 since if it goes
    		#doewn there is nothing for us to do
		return 0
	fi
	
	if [ $route_eth0 = "down" ]; then
		logit_debug "route 8.8.8.8 not ok"
		rc=`ip route show | grep "nexthop via"`
		rc1=`ip route show | grep "default.*$eth0_dev"`
		rc2=`ip route show | grep "default"`
		if [ "$rc" != "" ] || [ "$rc1" != "" ] || [ "$rc2" = "" ]; then
			logit_crit "Error: eth0 route failed: $def_route_eth0. Taking down the route"
			runit "ip route del default"
			runit "route add default gw $def_route_eth1"
			runit "ip route flush cache"
			ntp_restart
		fi
	fi

	return 0	

}

while [ 1 ]
do
	check_for_multiple_default_routes
	resolv_check
	logit_debug "Sleeping"
	sleep 1
done


on_die


exit 0