#!/bin/bash

# (C)DatuX - Released under GPL
# Asterisk stresstest script
# Tries to discover asterisk unstability and regressions with sipp and special dialplans.
# Usefull to find out you're using the wrong gcc/glibc/compiler options.
# Configuration on target system is replaced with ./asteriskstress/asterisk/*


source testlib || exit 1

for PAR in $*; do eval "$PAR=1" &>/dev/null; done

if [ "$ASTERISK_SKIP_STRESS" ]; then
	echo "Skipping asterisk stress test..."
	exit 0
fi


DESCRIPTION1="Skip test if we dont have the right option in SYN-3"
CMD="
	if [ -e /etc/syn3-version ]; then
		grep '\"V\"' /etc/webint/REG || exit 1
	else
		exit 0
	fi
"
VERIFYEXIT=0
if ! sshtest; then
	echo "No voiceserver, test skipped"
	exit 0
fi


DESCRIPTION1="Check if box has enough memory (512mb)."
CMD='[ `grep MemTotal /proc/meminfo|egrep -o "([0-9]+)"` -ge 512000 ]'
VERIFYEXIT=0
sshtest || exit 1

#Execute a asterisk cli command
asterisk_cmd()
{
	CMD="asterisk -r -x '$ASTERISK_CMD'"
	DESCRIPTION1="Asterisk CLI command: $ASTERISK_CMD"
	VERIFYEXIT=0
	VERIFYPOSITIVE="$ASTERISK_VERIFYPOSTIVE"
	VERIFYNEGATIVE="$ASTERISK_VERIFYNEGATIVE"
	sshtest 	
	return $?
}

#Check if asterisk is crashed or has logged errors
crashcheck()
{
	DESCRIPTION1="Check for asterisk crashes/errors... ($CASE)"
	CMD="
		echo 'Asterisk core is still responding via CLI?'
		echo 'If not responding, either the load is too high or asterisk cant create new threads because of a glibc/gcc incompatiblity. You also need 512 mb memory!'
		asterisk -r -x 'core show uptime' | grep 'System uptime' || exit 1
		
		echo 'Errors in asterisk logs?'
		cat /var/log/asterisk/full |
			grep -v 'Voicemail will be lost.' |
			grep -v 'RTCP .. transmission error' |
			grep ERROR && exit 1
	"
	VERIFYEXIT=0
	sshtest || exit 1
}

#Wait untill asterisk crashes or sipp exists, or a timeout has occurred, 
#or on a couple of other optional checks..
crashwait()
{
	(( END=$SECONDS+$MAXTIME ))

	while [ $SECONDS -lt $END ]; do
		crashcheck || exit 1

		if [ "$ASTERISK_VERIFYPOSTIVE" ]; then
			DESCRIPTION1="Asterisk status check ($CASE)"
			asterisk_cmd || exit 1
			#we only check positive things ONE time, because at the end of the test it might be gone
			if [ "$ASTERISK_VERIFYPOSTIVE" ]; then
				ASTERISK_VERIFYPOSTIVE=
			fi
		fi

		if [ "$UAC_NOERRORS" == 1 ]; then
			DESCRIPTION1="Sipp client has log no errors? ($CASE)"
			CMD="	
				test -s /root/asteriskstress/*_*_*errors.log || exit 1
			"
			VERIFYEXIT=1
			sshtest || exit 1
		fi

		DESCRIPTION1="Sipp client has ended? ($CASE)"
		CMD="
			pgrep -f '^sipp.*trace_screen' || exit 1
		"
		VERIFYEXIT=1
		sshtest && return 0

		echo "Waiting until sipp client is done..., timeout after $MAXTIME seconds. ($CASE)"
		sleep 5
	done
	echo "Timeout! Test didnt complete in time!"
	exit 1
}

#runs SIPP test scenario for a certain time and check if nothing crashes or acts weird.
sipprun()
{
	#Auto-resume part.
	#Because these tests take LONG you can run the script with the 'resume' option.
	RESUMEFILE=".asteriskstress-$IP/`echo \"$CASE\" | md5sum|cut -f1 -d' '`"
	if [ -e $RESUMEFILE ]; then
		echo "SKIPPING CASE: $CASE"
		return 0
	fi
	
	COUNT=0
	#restart sipp client in case it crashes (happens a lot :()
	while true; do
		DESCRIPTION1="Start sipp test client... ($CASE)"
		CMD="
			ulimit -c unlimited
			rm /root/asteriskstress/core*
			rm /root/asteriskstress/*_*_*.log
			ulimit -n 100000
			cd /root/asteriskstress
			sipp -bind_local -i 127.0.0.1 -mi 127.0.0.1 -ci 127.0.0.1 -recv_timeout 60000 -send_timeout 60000 -bg -nostdin -trace_shortmsg -trace_screen -trace_err $PARS 
			pgrep -f '^sipp.*trace_screen' || exit 1
		"
		VERIFYEXIT=0
		sshtest || exit 1
	
		#give sipp a while to start connecting
		sleep 5
	
		#wait and see what happens...
		crashwait 
	
		DESCRIPTION1="sipp client has exited cleanly, without crash? ($CASE)"
		CMD="
			pgrep -f '^sipp.*trace_screen' && exit 1
			test -s /root/asteriskstress/*_*_screen.log || exit 1
		"
		VERIFYEXIT=0
		sshtest && break # everything went fine...break the loop
	
		(( COUNT++ ))
		echo "### sipp client crashed! restarting test! attempt number $COUNT ###"
	done
		
	DESCRIPTION1="sipp server still running? ($CASE)"
	CMD="
		pgrep -f '^sipp.*uas' || exit 1
	"
	VERIFYEXIT=0
	sshtest || exit 1

	#let asterisk restart convienently, to check for hanging threads etc..
	if [ "$ASTERISK_RESTARTCONVENIENT" == "1" ]; then
		restart_convenient
	fi


	#Done, mark resume file
	touch $RESUMEFILE
}

restart_all()
{
	VERIFYEXIT=0
	DESCRIPTION1="Copy config, clear environment, restart asterisk"
	CMD="
		svc -d /service/asterisk &>/dev/null
		killall -9 sipp
		killall -9 safe_asterisk asterisk
		sleep 1; 
		rm /etc/asterisk/*
		cp /root/asteriskstress/asterisk/* /etc/asterisk || exit 1
		rm /var/log/asterisk/full; 
		rm /home/system/asterisk/spool/voicemail/default/1002/INBOX/*
		rm /home/system/asterisk/spool/voicemail/default/1002/INBOX/.*
		if which svc &>/dev/null; then
			#start on syn3:
			echo 'Starting SYN-3 asterisk'
			svc -do /service/asterisk
		else
			#start manually
			echo 'Starting manual asterisk'
			ulimit -n 100000
			asterisk
		fi
		sleep 3
	"
	sshtest || exit 1
	
	
	DESCRIPTION1="Restart sipp server"
	CMD="
		ulimit -n 100000
		sipp -sn uas -cp 18888 -bind_local -i 127.0.0.2 -mi 127.0.0.2 -ci 127.0.0.2 -mp 16000 -p 15060 -bg -nostdin
		pgrep -f '^sipp.*uas' || exit 1
	"
	VERIFYEXIT=0
	sshtest || exit 1

	#turn of verbose and debugging, to increase performcance and decrease log size
	ASTERISK_CMD="core set verbose 0"
	ASTERISK_VERIFYPOSTIVE=
	asterisk_cmd || exit 1
	
	ASTERISK_CMD="core set debug 0"
	ASTERISK_VERIFYPOSITIVE=
	asterisk_cmd || exit 1

}

#Let asterisk restart when convienent. 
#This way we can check for hanging threads.
restart_convenient()
{
	CMD="ls -il /var/run/asterisk/asterisk.* > /tmp/oldstatus"
	sshtest || exit 1

	ASTERISK_CMD="restart when convenient"
	asterisk_cmd  || exit 1

	(( END=$SECONDS+ $MAXTIME ))

	while [ $SECONDS -lt $END ]; do
		DESCRIPTION1="Has asterisk restarted yet? ($CASE)"
		CMD="
			ls -il /var/run/asterisk/asterisk.* > /tmp/newstatus
			cmp /tmp/oldstatus /tmp/newstatus && exit 1
		"
		VERIFYEXIT=0
		sshtest && return 0
		echo "Waiting for asterisk to restart when convenient.."
		sleep 5
	done
	echo "Timeout! Test didnt complete in time!"
	exit 1

}

################################### PREPERATION #################################

if ! [ "$resume" ]; then
	rm -r .asteriskstress-$IP &>/dev/null
fi
mkdir .asteriskstress-$IP &>/dev/null

scptest "asteriskstress" || exit 1



MAXTIME=3600

################### TEST CASES ##########################################

#start with a clean asterisk environment
restart_all




CASE="Meetme test without audio. Asterisk will be unable to create new threads if you have the wrong glibc/gcc/kernelmodule. This also tests the dahdi kernel module."
PARS="-sn uac -s 1000 localhost -users 1000 -m 4000 -d 5000"
ASTERISK_CMD="show channels"
ASTERISK_VERIFYPOSTIVE="MeetMe"
UAC_NOERRORS=
ASTERISK_RESTARTCONVENIENT= #takes too long for meetme
sipprun 
#forcefully restart everything, to recover from the heavy meetme test. (takes ages otherwise)
restart_all

CASE="Meetme test with audio. Causes 100% softinterupts that freese whole box if you have the wrong glibc/gcc/kenelmodule. (This also tests the dahdi kernel module.)"
PARS="-sn uac_pcap -s 1000 localhost -r 20 -m 2000 "
ASTERISK_CMD="show channels"
ASTERISK_VERIFYPOSTIVE="MeetMe"
UAC_NOERRORS=
ASTERISK_RESTARTCONVENIENT= #takes too long for meetme
sipprun 
# forcefully restart everything, to recover from the heavy meetme test. (takes ages otherwise)
restart_all


CASE="Parking lot test"
PARS="-sn uac_pcap -s 1001 localhost -r 20 -m 2000 "
ASTERISK_CMD="show channels"
ASTERISK_VERIFYPOSTIVE="Park"
UAC_NOERRORS=
ASTERISK_RESTARTCONVENIENT=1
sipprun 


CASE="Voicemail test"
PARS="-sn uac_pcap -s 1002 localhost -r 10 -m 1000 "
ASTERISK_CMD="show channels"
ASTERISK_VERIFYPOSTIVE="VoiceMail"
UAC_NOERRORS=
ASTERISK_RESTARTCONVENIENT=1
sipprun 



#call/second rate and total calls for codec tests
#keep this low, because sipp client is checked for errors in this test.
RATE=2
CALLS=200

CASE="Codec g729 transcoding"
PARS="-sn uac_pcap -s 209 localhost -r $RATE -m $CALLS "
ASTERISK_CMD="sip show channels"
ASTERISK_VERIFYPOSTIVE="g729"
UAC_NOERRORS=1
sipprun 

CASE="Codec g723 transcoding"
PARS="-sn uac_pcap -s 201 localhost -r 1 -m 200 "
ASTERISK_CMD="sip show channels"
ASTERISK_VERIFYPOSTIVE="g723"
UAC_NOERRORS=1
sipprun 

# CASE="Codec g722 transcoding"
# PARS="-sn uac_pcap -s 213 localhost -r 1 -m 200 "
# ASTERISK_CMD="sip show channels"
# ASTERISK_VERIFYPOSTIVE="g722"
# UAC_NOERRORS=1
# sipprun 


CASE="Codec g726 transcoding"
PARS="-sn uac_pcap -s 212 localhost -r $RATE -m $CALLS "
ASTERISK_CMD="sip show channels"
ASTERISK_VERIFYPOSTIVE="g726"
UAC_NOERRORS=1
ASTERISK_RESTARTCONVENIENT=1
sipprun 

# CASE="Codec ilbc transcoding"
# PARS="-sn uac_pcap -s 211 localhost -r $RATE -m $CALLS "
# ASTERISK_CMD="sip show channels"
# ASTERISK_VERIFYPOSTIVE="ilbc"
# UAC_NOERRORS=1
# sipprun 

# CASE="Codec speex transcoding"
# PARS="-sn uac_pcap -s 210 localhost -r $RATE -m $CALLS "
# ASTERISK_CMD="sip show channels"
# ASTERISK_VERIFYPOSTIVE="speex"
# UAC_NOERRORS=1
# sipprun 


CASE="Codec lpc10 transcoding"
PARS="-sn uac_pcap -s 208 localhost -r $RATE -m $CALLS "
ASTERISK_CMD="sip show channels"
ASTERISK_VERIFYPOSTIVE="lpc10"
ASTERISK_RESTARTCONVENIENT=1
UAC_NOERRORS=1
sipprun 

CASE="Codec slin transcoding"
PARS="-sn uac_pcap -s 207 localhost -r $RATE -m $CALLS "
ASTERISK_CMD="sip show channels"
ASTERISK_VERIFYPOSTIVE="slin"
ASTERISK_RESTARTCONVENIENT=1
UAC_NOERRORS=1
sipprun 

CASE="Codec adpcm transcoding"
PARS="-sn uac_pcap -s 206 localhost -r $RATE -m $CALLS "
ASTERISK_CMD="sip show channels"
ASTERISK_VERIFYPOSTIVE="adpcm"
ASTERISK_RESTARTCONVENIENT=1
UAC_NOERRORS=1
sipprun 

CASE="Codec g726aal2 transcoding"
PARS="-sn uac_pcap -s 205 localhost -r $RATE -m $CALLS "
ASTERISK_CMD="sip show channels"
ASTERISK_VERIFYPOSTIVE="g726aal2"
ASTERISK_RESTARTCONVENIENT=1
UAC_NOERRORS=1
sipprun 


CASE="Codec alaw (sipp pcap uses this too, so now actual transcoding is done)"
PARS="-sn uac_pcap -s 204 localhost -r $RATE -m $CALLS "
ASTERISK_CMD="sip show channels"
ASTERISK_VERIFYPOSTIVE="alaw"
ASTERISK_RESTARTCONVENIENT=1
UAC_NOERRORS=1
sipprun 

CASE="Codec ulaw transcoding"
PARS="-sn uac_pcap -s 203 localhost -r $RATE -m $CALLS "
ASTERISK_CMD="sip show channels"
ASTERISK_VERIFYPOSTIVE="ulaw"
ASTERISK_RESTARTCONVENIENT=1
UAC_NOERRORS=1
sipprun 

CASE="Codec gsm transcoding"
PARS="-sn uac_pcap -s 202 localhost -r $RATE -m $CALLS "
ASTERISK_CMD="sip show channels"
ASTERISK_VERIFYPOSTIVE="gsm"
ASTERISK_RESTARTCONVENIENT=1
UAC_NOERRORS=1
sipprun 


#probleem 1: na een test die zowel voicemail als normale nummers belt, blijven er een aantal Dials hangen.
#dus alleen met 1002 en 203 in de stress.csv
CASE="Random test of everything (heavy and takes a lot of time)"
PARS="-sf uac_stress.xml -inf stress.csv localhost -users 100 -m 10000 "
ASTERISK_CMD=
ASTERISK_VERIFYPOSTIVE=
ASTERISK_RESTARTCONVENIENT= #kan niet, ivm probleem 1
UAC_NOERRORS=
sipprun 


echo "########################################################"
echo "All tests successfull"