#!/bin/bash

##################################################
# Author: pao
# Date: Nov 1 2008
#
# Script to set off multiple monitors on a *NIX Type Server
#
# Usage
# -----
# $0 <base output filename> <sample interval> <no samples>
#
# Possible Extentions & Limitations:
# 1. Run the most appropriate monitors only - need to check pros & cons of all options
# 2. Include a "default" option and then user-defined options for which monitors to choose
# 
#
#
# Modification History
# --------------------
# Nov 26 2008 pao: 
#  Added "iostat" as a monitor to get device-specific information
#  Tided up format of "who" output to track all users and date
#
##################################################

hostname=`hostname` #Get hostname to place in output files

#useage message
if [ $# -eq 0 ]; then
echo -e "
Usage:

monitor-system <filename> <sample interval> <no samples>

e.g.
nohup ./monitor-system test1 20 100 &

will initialise system monitors taking 100 samples at 20 second intervals.
Monitors are:

top
sar
iostat (x 2)
ps
who
vmstat

NOTE NOTE NOTE:
Use \"nohup\" before command 
Execute in background with \" &\"
This is to allow shell to continue execution if connection is broken
"
exit
fi

#variable assignments and calculations
prefix=$1"."$hostname

no_samples=$3
sleeptime=$2

#create output filenames & remove if they already exits
processfile=$prefix".top"
processnamefile=$prefix".processnames"
sarfilename=$prefix".sar"               #sarfile has to be .sar for all output to be generated
wfilename=$prefix".users"
iostatxfilename=$prefix".iostat-x"
iostatpfilename=$prefix".iostat-p"
vmstatfilename=$prefix".vmstat"

if [ -f $processfile ]; then rm $processfile; fi
if [ -f $processnamefile ]; then rm $processnamefile; fi
if [ -f $sarfilename ]; then rm $sarfilename; fi
if [ -f $wfilename ]; then rm $wfilename; fi
if [ -f $iostatxfilename ]; then rm $iostatxfilename; fi
if [ -f $iostatpfilename ]; then rm $iostatpfilename; fi
if [ -f $vmstatfilename ]; then rm $vmstatfilename; fi

#Calculate a suitable sample interval at which to run "who" and "ps"
totaltime=`echo "$no_samples*$sleeptime" | bc`
sleep_ps=300
   #Hardcoded fifures here, not necessarily optimal, can change at a later date
if [ $totaltime -gt 3600 ]; then           #If totaltime > 1 hour, run ps every 5 mins
 sleep_ps=300;
else if [ $totaltime -gt 100 ]; then       #If 100s < totaltime < 1 hour, run 100 ps samples
 sleep_ps=`echo  "$totaltime / 100" | bc`;
else                                       #If totaltime < 100s then run ps at same rate as everything else.
 sleep_ps=$sleeptime;
fi
fi

#run monitors and append output to files
#    ----- top -----
nohup top -b -c -d$sleeptime -n$no_samples  | awk '{if (match($0,"top -")){c=1;print $0}else if(c>=7&&c<=32){print $0};c++}' >> $processfile &

#    ----- sar -----
nohup sar -o $sarfilename $sleeptime $no_samples >& /dev/null &

#    ----- iostat -----
nohup iostat -t -x     $sleeptime $no_samples > $iostatxfilename &
nohup iostat -t -k -p ALL $sleeptime $no_samples > $iostatpfilename &

#    ----- vmstat ------
nohup vmstat -n $sleeptime $no_samples > $vmstatfilename &

#echo some information here:
top=`echo "nohup top -b -d$sleeptime -n$no_samples"`  
sar=`echo "sar -o $sarfilename $sleeptime $no_samples >& /dev/null &"`
iostatx=`echo "nohup iostat -t -x $sleeptime $no_samples > $iostatxfilename &"`
iostatp=`echo "nohup iostat -t -p ALL $sleeptime $no_samples > $iostatpfilename &"`
vmstatCmd=`echo "nohup vmstat -n $sleeptime $no_samples > $vmstatfilename &"`
ps=`echo "(date; ps -e -opid,ppid,comm,args) >> $processnamefile"`
who=`echo "(date +'%H:%M:%S %b %d %y %Z';who)  >> $wfilename "`

echo -e " \n The following monitors have been set off: \n
 $top 
 $sar 
 $iostatx
 $iostatp
 $vmstatCmd
 $ps
 $who \n \n
 Also, Please Note: \"user count monitor\" and \"ps\" will be run at $sleep_ps (second) intervals"

#    ----- ps  & w -----
#run ps & w only rarely to sample all process names: this is to reduce size text file
#
#who:
# Combine with cusomeised "date" to get a time-stamped output format
# In awk, use associative arrays to get a useful output format 
time=0
while (( $time<=$totaltime )); do
 (date; ps -e -opid,ppid,comm,args) >> $processnamefile;
 (date +'%H:%M:%S %b %d %y %Z';who) | \
     awk '{if(FNR==1){date=$0}else{w[$1]+=1}};END{str=date" USERS: ";for (i in w){str=str" "i" "w[i]};print str}' >> $wfilename;
 time=$( echo $sleep_ps + $time | bc);
 sleep $sleep_ps;
done