#!/bin/bash # PROGRAM: backup.notify.zabbix # PURPOSE: Notify Zabbix of backup start and end # CREATOR: Patrick Swanson (ips@qad.com) # CREATED: July 17, 2013 # CHANGES: # 07/17/13 ips OND-1646 Created based on logic from Derek # 10/27/16 ips OND-5902 Add kill SIGUSR1 of progress_agent and call of # force_passive_check.sh # 01/10/17 ips OND-6287 Add sleep 300 to give Tomcat, AS, WS, etc a chance to # come up. Increase sleep between kill sigusr2 and Icinga recheck # to 45 seconds per environment. # 05/16/17 ips OND-7084 Get Zabbix server IPs from progress_agent config files # 09/05/17 ips OND-7416 Change to use sudo to run clustat so YAB systems work. # 09/26/17 ips OND-7426 Add pause to give progress_agent time to die if it wants # 09/29/17 ips OND-7561 Check for clustat before running it # 06/05/18 ips OND-3493 Set for all servers in the same customer/environment. # 08/27/18 ips OND-8663 Fix handling of hostname for zabbix_sender calls # 10/05/18 ips OND-8793 Eliminate all use of FQDN. Increase curl timeout. # 10/30/18 ips OND-8875 Moved monitoring restart logic to new script # start_monitoring_scan.sh and run via sudo # Re-add sudo to clustat which was lost in rev 1660. help() { # Called by getopts echo " Usage: `basename $0` [OPTIONS] -b Begin backups -e End backups (default)" exit 0 } # Default to end downtime ACTION=0 # Parse command line options while getopts "eb" opts do case "$opts" in b) ACTION=1 ;; e) ACTION=0 ;; \?) help;; esac done shift $(( $OPTIND - 1 )) cd $(dirname $0) env=`pwd | cut -f 2 -d /` # Identify Zabbix server for this environment for FILE in $(grep -l ^enviro=$env /usr/local/etc/progress_agent*.cfg) do ZABIP=`grep ^server= $FILE | awk -F= '{print $2}'` QADENV=`grep ^qadenv= $FILE | awk -F= '{print $2}'` if [ "$ZABIP" != "" ] then if [ "$zabbixservers" == "" ] then zabbixservers=$ZABIP else case $ZABIP in *${zabbixservers}* ) ;; * ) zabbixservers="$zabbixservers $ZABIP" esac fi fi done # Identify all Zabbix servers if [ "$zabbixservers" == "" ] then for FILE in $(ls /usr/local/etc/progress_agent*.cfg) do ZABIP=`grep ^server= $FILE | awk -F= '{print $2}'` if [ "$ZABIP" != "" ] then if [ "$zabbixservers" == "" ] then zabbixservers=$ZABIP else case $ZABIP in *${zabbixservers}* ) ;; * ) zabbixservers="$zabbixservers $ZABIP" esac fi fi done fi # Default Zabbix servers if [ "$zabbixservers" == "" ] then zabbixservers="216.58.134.132 216.58.134.142" fi # Get host name myhost=`hostname -s` # Trim off leading #- for physicals case $myhost in 1*|2*|3*|4*|5*|6*|7*|8*|9*) case $myhost in *-*) myhost=`echo $myhost | awk -F - '{print $2}'` ;; esac ;; esac # Identify all servers in the portal for the same customer and environment if [ "$QADENV" == "prod" ] then PORTALENV="Production" else PORTALENV=$QADENV fi TMPFILE="/dev/shm/backup.notify.zabbix.send" # Send file missing or this is start-of-backup - need to build server list if [ ! -f $TMPFILE ] || [ "$ACTION" == "1" ] then # Get the web app password if [ -f /usr/remote/odutils.src/user_mgmt/.liferay.svc ] then WEBPWD=`cat /usr/remote/odutils.src/user_mgmt/.liferay.svc` elif [ -f /opt/odutils/user_mgmt/.liferay.svc ] then WEBPWD=`cat /opt/odutils/user_mgmt/.liferay.svc` else WEBPWD='' fi ################################################################### # Query the web app and identify which hosts to set in backup mode ################################################################### MYDBS="" MYALL="" LIVEDBS="" LIVEALL="" PRELIVEDBS="" PRELIVEALL="" POSTDBS="" POSTALL="" MYSTAT="" myhost=`hostname -s | tr '[:upper:]' '[:lower:]'` # Trim off leading #- for physicals case $myhost in 1*|2*|3*|4*|5*|6*|7*|8*|9*) case $myhost in *-*) myhost=`echo $myhost | awk -F - '{print $2}'` ;; esac ;; esac # Identify if cluster and adjust if [ -f /usr/sbin/clustat ] then clustername=`sudo /usr/sbin/clustat | grep service | awk '{print $1}' | awk -F: '{print $2}'` myhost=$clustername # Allow up to 3 (cluster and 2 nodes) MAXLDB=3 else clustername="" MAXLDB=1 fi if [ "$WEBPWD" != "" ] then MYDATA=`curl -m 15 https://myqadcloud.qad.com/api/jsonws/od-environment-portlet.environment/get-customer-environments-by-server -u liferay.svc:$WEBPWD -d server="$myhost" -d type="$PORTALENV" 2>/dev/null | sed "s/ //g"` # Get all servers for this server's EU/environment from portal for SRVR in $(echo $MYDATA | sed "s/,/\n/g" | grep -e '"server":"' -e '"prePostLive":"' -e '"QADVersion":"' | sed 's/"//g') do # Parse data case $SRVR in server*) SRVRNAME=`echo $SRVR | awk -F: '{print $2}' | tr '[:upper:]' '[:lower:]'` ;; prePostLive*) SRVRSTAT=`echo $SRVR | awk -F: '{print $2}' | tr '[:upper:]' '[:lower:]'` continue ;; QADVersion*) QADVER=`echo $SRVR | awk -F: '{print $2}' | tr '[:upper:]' '[:lower:]'` continue ;; *) continue ;; esac # Get list of this customer's matching envrionment MFGPRO servers if [ "$QADVER" != "" ] then # Note status of this server if [ "$SRVRNAME" == "$myhost" ] then MYSTAT=$SRVRSTAT fi # Group by pre/post/transitioning if [ "$SRVRSTAT" == "pre" ] || \ [ "$SRVRSTAT" == "pre-live" ] then PRELIVEDBS="$SRVRNAME $PRELIVEDBS" elif [ "$SRVRSTAT" == "live" ] || \ [ "$SRVRSTAT" == "live-hypercare" ] || \ [ "$SRVRSTAT" == "live-project" ] || \ [ "$SRVRSTAT" == "post" ] then LIVEDBS="$SRVRNAME $LIVEDBS" elif [ "$SRVRSTAT" == "live-transition" ] then POSTDBS="$SRVRNAME $POSTDBS" fi fi # Get list of ALL of this customer's matching environment servers # Group by pre/post/transitioning if [ "$SRVRSTAT" == "pre" ] || \ [ "$SRVRSTAT" == "pre-live" ] then PRELIVEALL="$SRVRNAME $PRELIVEALL" elif [ "$SRVRSTAT" == "live" ] || \ [ "$SRVRSTAT" == "live-hypercare" ] || \ [ "$SRVRSTAT" == "live-project" ] || \ [ "$SRVRSTAT" == "post" ] then LIVEALL="$SRVRNAME $LIVEALL" elif [ "$SRVRSTAT" == "live-transition" ] then POSTALL="$SRVRNAME $POSTALL" fi SRVRNAME="" SRVRSTAT="" QADVER="" done # Pick list of servers matching this server's status if [ "$MYSTAT" != "" ] then if [ "$MYSTAT" == "pre" ] || \ [ "$MYSTAT" == "pre-live" ] then MYDBS=$PRELIVEDBS MYALL=$PRELIVEALL elif [ "$MYSTAT" == "live" ] || \ [ "$MYSTAT" == "live-hypercare" ] || \ [ "$MYSTAT" == "live-project" ] || \ [ "$MYSTAT" == "post" ] then MYDBS=$LIVEDBS MYALL=$LIVEALL elif [ "$MYSTAT" == "live-transition" ] then POSTALL="$MYSTAT $POSTALL" MYDBS=$POSTDBS MYALL=$POSTALL fi else # Cannot match so only manage monitoring for this server MYDBS=$myhost MYALL=$myhost fi LDBCNT=0 #echo $MYDBS #echo $MYALL # Count DB servers selected for ENTRY in $MYDBS do let "LDBCNT=LDBCNT+1" done # echo Found $LDBCNT DB server - max is $MAXLDB # echo "$myhost related by $MYSTAT to $MYALL" # Do all customer server for this environment or just this one if [ $LDBCNT -le $MAXLDB ] then ENVSRVRS=$MYALL else ENVSRVRS=$myhost fi else ENVSRVRS=$myhost fi fi # Things to do before sending notification of end of backup if [ $ACTION == "0" ] then # Moved logic to new script so only 1 sudo is needed if [ -f /usr/remote/odutils/start_monitoring_scan.sh ] then sudo /usr/remote/odutils/start_monitoring_scan.sh "$ENVSRVRS" fi fi ####################################################### # Notifify Zabbix that backups are starting or ending # ####################################################### if [ -f $TMPFILE ] then if [ "$ACTION" == "0" ] then # Convert start to stop sed -i "s/ 1$/ 0/" $TMPFILE else # Clear file for new backup run /bin/rm -f $TMPFILE fi fi # Either starting backups or file doesn't exist if [ ! -f $TMPFILE ] then # Populate temp file with all host/key/value sets for SRVR in $ENVSRVRS do echo "$SRVR oe.$env.backup.status $ACTION" >> $TMPFILE if [ "$env" == "qond" ] then echo "$SRVR oe.$QADENV.backup.status $ACTION" >> $TMPFILE fi done fi # Send to each identified Zabbix server for ZABIP in $zabbixservers do /usr/local/bin/zabbix_sender -z $ZABIP -i $TMPFILE done # Remove temp file if [ "$ACTION" == "0" ] && [ -f $TMPFILE ] then /bin/rm -f $TMPFILE fi