#!/bin/bash
# PROGRAM: backup.notify.zabbix
# PURPOSE: Notify Zabbix of backup start and end
# CREATOR: Patrick Swanson (ips@qad.com)
# CREATED: July 17, 2013
# CHANGES:
# 07/17/13 ips OND-1646 Created based on logic from Derek
# 10/27/16 ips OND-5902 Add kill SIGUSR1 of progress_agent and call of
#              force_passive_check.sh
# 01/10/17 ips OND-6287 Add sleep 300 to give Tomcat, AS, WS, etc a chance to 
#              come up. Increase sleep between kill sigusr2 and Icinga recheck
#              to 45 seconds per environment.
# 05/16/17 ips OND-7084 Get Zabbix server IPs from progress_agent config files
# 09/05/17 ips OND-7416 Change to use sudo to run clustat so YAB systems work.
# 09/26/17 ips OND-7426 Add pause to give progress_agent time to die if it wants
# 09/29/17 ips OND-7561 Check for clustat before running it
# 06/05/18 ips OND-3493 Set for all servers in the same customer/environment.
# 08/27/18 ips OND-8663 Fix handling of hostname for zabbix_sender calls
# 10/05/18 ips OND-8793 Eliminate all use of FQDN. Increase curl timeout.
# 10/30/18 ips OND-8875 Moved monitoring restart logic to new script
#              start_monitoring_scan.sh and run via sudo
#              Re-add sudo to clustat which was lost in rev 1660.

help() {
    # Called by getopts
echo "    Usage: `basename $0` [OPTIONS]
      -b     Begin backups
      -e     End backups (default)"
    exit 0
}

# Default to end downtime
ACTION=0

# Parse command line options
while getopts "eb" opts
do
    case "$opts" in
        b) ACTION=1 ;;
        e) ACTION=0
           ;;
        \?)  help;;
    esac
done
shift $(( $OPTIND - 1 ))

cd $(dirname $0)
env=`pwd | cut -f 2 -d /`

# Identify Zabbix server for this environment
for FILE in $(grep -l ^enviro=$env /usr/local/etc/progress_agent*.cfg)
do
    ZABIP=`grep ^server= $FILE | awk -F= '{print $2}'`
    QADENV=`grep ^qadenv= $FILE | awk -F= '{print $2}'`
    if [ "$ZABIP" != "" ]
    then
        if [ "$zabbixservers" == "" ]
        then
            zabbixservers=$ZABIP
        else
            case $ZABIP in
                *${zabbixservers}* ) ;;
                * ) zabbixservers="$zabbixservers $ZABIP"
            esac
        fi
    fi
done

# Identify all Zabbix servers
if [ "$zabbixservers" == "" ]
then
    for FILE in $(ls /usr/local/etc/progress_agent*.cfg)
    do
        ZABIP=`grep ^server= $FILE | awk -F= '{print $2}'`
        if [ "$ZABIP" != "" ]
        then
            if [ "$zabbixservers" == "" ]
            then
                zabbixservers=$ZABIP
            else
                case $ZABIP in
                    *${zabbixservers}* ) ;;
                    * ) zabbixservers="$zabbixservers $ZABIP"
                esac
            fi
        fi
    done
fi

# Default Zabbix servers
if [ "$zabbixservers" == "" ]
then
    zabbixservers="216.58.134.132 216.58.134.142"
fi

# Get host name
myhost=`hostname -s`

# Trim off leading #- for physicals
case $myhost in
    1*|2*|3*|4*|5*|6*|7*|8*|9*)
        case $myhost in
            *-*) myhost=`echo $myhost | awk -F - '{print $2}'` ;;
        esac
        ;;
esac

# Identify all servers in the portal for the same customer and environment
if [ "$QADENV" == "prod" ]
then
    PORTALENV="Production"
else
    PORTALENV=$QADENV
fi

TMPFILE="/dev/shm/backup.notify.zabbix.send"

# Send file missing or this is start-of-backup - need to build server list
if [ ! -f $TMPFILE ] || [ "$ACTION" == "1" ]
then
    # Get the web app password
    if [ -f /usr/remote/odutils.src/user_mgmt/.liferay.svc ]
    then
        WEBPWD=`cat /usr/remote/odutils.src/user_mgmt/.liferay.svc`
    elif [ -f /opt/odutils/user_mgmt/.liferay.svc ]
    then
        WEBPWD=`cat /opt/odutils/user_mgmt/.liferay.svc`
    else
        WEBPWD=''
    fi

    ###################################################################
    # Query the web app and identify which hosts to set in backup mode
    ###################################################################
    MYDBS=""
    MYALL=""
    LIVEDBS=""
    LIVEALL=""
    PRELIVEDBS=""
    PRELIVEALL=""
    POSTDBS=""
    POSTALL=""
    MYSTAT=""
    myhost=`hostname -s | tr '[:upper:]' '[:lower:]'`

    # Trim off leading #- for physicals
    case $myhost in
        1*|2*|3*|4*|5*|6*|7*|8*|9*)
            case $myhost in
                *-*) myhost=`echo $myhost | awk -F - '{print $2}'` ;;
            esac
            ;;
    esac

    # Identify if cluster and adjust
    if [ -f /usr/sbin/clustat ]
    then
        clustername=`sudo /usr/sbin/clustat | grep service | awk '{print $1}' | awk -F: '{print $2}'`
        myhost=$clustername
        # Allow up to 3 (cluster and 2 nodes)
        MAXLDB=3
    else
        clustername=""
        MAXLDB=1
    fi

    if [ "$WEBPWD" != "" ]
    then
        MYDATA=`curl -m 15 https://myqadcloud.qad.com/api/jsonws/od-environment-portlet.environment/get-customer-environments-by-server -u liferay.svc:$WEBPWD -d server="$myhost" -d type="$PORTALENV" 2>/dev/null | sed "s/ //g"`

        # Get all servers for this server's EU/environment from portal
        for SRVR in $(echo $MYDATA | sed "s/,/\n/g" | grep -e '"server":"' -e '"prePostLive":"' -e '"QADVersion":"' | sed 's/"//g')
        do
            # Parse data
            case $SRVR in
                server*)
                    SRVRNAME=`echo $SRVR | awk -F: '{print $2}' | tr '[:upper:]' '[:lower:]'`
                    ;;
                prePostLive*)
                    SRVRSTAT=`echo $SRVR | awk -F: '{print $2}' | tr '[:upper:]' '[:lower:]'`
                    continue
                    ;;
                QADVersion*)
                    QADVER=`echo $SRVR | awk -F: '{print $2}' | tr '[:upper:]' '[:lower:]'`
                    continue
                ;;
                *)  continue
                    ;;
            esac

            # Get list of this customer's matching envrionment MFGPRO servers
            if [ "$QADVER" != "" ]
            then

                # Note status of this server
                if [ "$SRVRNAME" == "$myhost" ]
                then
                    MYSTAT=$SRVRSTAT
                fi

                # Group by pre/post/transitioning
                if [ "$SRVRSTAT" == "pre" ] || \
                   [ "$SRVRSTAT" == "pre-live" ]
                then
                    PRELIVEDBS="$SRVRNAME $PRELIVEDBS"
                elif [ "$SRVRSTAT" == "live" ] || \
                     [ "$SRVRSTAT" == "live-hypercare" ] || \
                     [ "$SRVRSTAT" == "live-project" ] || \
                     [ "$SRVRSTAT" == "post" ]
                then
                    LIVEDBS="$SRVRNAME $LIVEDBS"
                elif [ "$SRVRSTAT" == "live-transition" ]
                then
                    POSTDBS="$SRVRNAME $POSTDBS"
                fi
            fi
            # Get list of ALL of this customer's matching environment servers
            # Group by pre/post/transitioning
            if [ "$SRVRSTAT" == "pre" ] || \
               [ "$SRVRSTAT" == "pre-live" ]
            then
                PRELIVEALL="$SRVRNAME $PRELIVEALL"
            elif [ "$SRVRSTAT" == "live" ] || \
                 [ "$SRVRSTAT" == "live-hypercare" ] || \
                 [ "$SRVRSTAT" == "live-project" ] || \
                 [ "$SRVRSTAT" == "post" ]
            then
                LIVEALL="$SRVRNAME $LIVEALL"
            elif [ "$SRVRSTAT" == "live-transition" ]
            then
                POSTALL="$SRVRNAME $POSTALL"
            fi

            SRVRNAME=""
            SRVRSTAT=""
            QADVER=""
        done

        # Pick list of servers matching this server's status
        if [ "$MYSTAT" != "" ]
        then
            if [ "$MYSTAT" == "pre" ] || \
               [ "$MYSTAT" == "pre-live" ]
            then
                MYDBS=$PRELIVEDBS
                MYALL=$PRELIVEALL
            elif [ "$MYSTAT" == "live" ] || \
                 [ "$MYSTAT" == "live-hypercare" ] || \
                 [ "$MYSTAT" == "live-project" ] || \
                 [ "$MYSTAT" == "post" ]
            then
                MYDBS=$LIVEDBS
                MYALL=$LIVEALL
            elif [ "$MYSTAT" == "live-transition" ]
            then
                POSTALL="$MYSTAT $POSTALL"
                MYDBS=$POSTDBS
                MYALL=$POSTALL
            fi
        else
            # Cannot match so only manage monitoring for this server
            MYDBS=$myhost
            MYALL=$myhost
        fi

        LDBCNT=0
        #echo $MYDBS
        #echo $MYALL
        # Count DB servers selected
        for ENTRY in $MYDBS
        do
            let "LDBCNT=LDBCNT+1"
        done
        # echo Found $LDBCNT DB server - max is $MAXLDB
        # echo "$myhost related by $MYSTAT to $MYALL"

        # Do all customer server for this environment or just this one
        if [ $LDBCNT -le $MAXLDB ]
        then
            ENVSRVRS=$MYALL
        else
            ENVSRVRS=$myhost
        fi
    else
        ENVSRVRS=$myhost
    fi
fi

# Things to do before sending notification of end of backup
if [ $ACTION == "0" ]
then
    # Moved logic to new script so only 1 sudo is needed
    if [ -f /usr/remote/odutils/start_monitoring_scan.sh ]
    then
        sudo /usr/remote/odutils/start_monitoring_scan.sh "$ENVSRVRS"
    fi
fi

#######################################################
# Notifify Zabbix that backups are starting or ending #
#######################################################

if [ -f $TMPFILE ]
then
    if [ "$ACTION" == "0" ]
    then
        # Convert start to stop
        sed -i "s/ 1$/ 0/" $TMPFILE
    else
        # Clear file for new backup run
        /bin/rm -f $TMPFILE
    fi
fi

# Either starting backups or file doesn't exist
if [ ! -f $TMPFILE ]
then
    # Populate temp file with all host/key/value sets
    for SRVR in $ENVSRVRS
    do
        echo "$SRVR oe.$env.backup.status $ACTION" >> $TMPFILE
        if [ "$env" == "qond" ]
        then
            echo "$SRVR oe.$QADENV.backup.status $ACTION" >> $TMPFILE
        fi
    done
fi

# Send to each identified Zabbix server
for ZABIP in $zabbixservers
do
    /usr/local/bin/zabbix_sender -z $ZABIP -i $TMPFILE
done

# Remove temp file
if [ "$ACTION" == "0" ] && [ -f $TMPFILE ]
then
    /bin/rm -f $TMPFILE
fi